{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Note: This is a visualized version of neural networks with multiple layers lib."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#load iris dataset for debugging\n",
    "from sklearn import datasets\n",
    "dataset = datasets.load_breast_cancer()\n",
    "#print(dataset['data'])    #input value of dataset\n",
    "#print(dataset['target'])   #True label of iris datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' \n",
    "In order to implement neural networks, which turns out that we are using multiple computing units(e.g. sigmoid or relu), \n",
    "it is required to initialize the parameter w randomly, whileb can be set to all zeros.\n",
    "'''\n",
    "\n",
    "# nn_structure is a tuple, indicating the number of layers and units in each layer.\n",
    "# for example, nn_structure = [4,4,3,1] indicates a neural network with 1 input layer, 2 hidden layers and one output unit.\n",
    "def nn_parameter_initialize(nn_structure):\n",
    "    # x_dim indicates the dimensions of input feature,a bias unit b is defaultly set.\n",
    "    parameter = {}\n",
    "    # Note: for ReLu unit, we use He initialization to faster convergence; for sigmoid in the output layer, do Xavier initialization. \n",
    "    for i in range(1,len(nn_structure)-1):\n",
    "        parameter['w'+str(i)] = np.random.randn(nn_structure[i],nn_structure[i-1])*np.sqrt(2./nn_structure[i-1])\n",
    "        parameter['b'+str(i)] = np.zeros((nn_structure[i],1))\n",
    "    L = len(nn_structure)-1\n",
    "    parameter['w'+str(L)] = np.random.randn(nn_structure[L],nn_structure[L-1])*np.sqrt(1./nn_structure[L-1])\n",
    "    parameter['b'+str(L)] = np.zeros((nn_structure[L],1))    \n",
    "    return parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x15c396f67f0>]"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAHiCAYAAAAnPo9XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3gc5bn38e9Pki03uRewLVuugCm2gzHNhGJMCy0cEiAhAUIOpBBCyXkPSc4hhJzk5CT0ACGE0JJQHJIQBwjYBptujI0LGNtyt+QmuUiustr9/jEjWIuVtNJKml3p/lyXLu3utHtm55l7nmeenZGZ4ZxzzrnUlBF1AM4555yrmydq55xzLoV5onbOOedSmCdq55xzLoV5onbOOedSmCdq55xzLoV5oq6DpK9Kmp5qy5U0W9I36xgmSY9J2iFpbstFGXfZ/5J0RWsuMxGSHpf0P1HH4VwsSV+UVCBpt6TxrbjcSI5rDZF0iqTCqONIVe06UUuaJOkdSaWStkt6W9IxAGb2ZzM7o7VjSnK5k4ApwGAzm9iMYR1A0m2S/hT7mZmdbWZPNNO8b0t2Ps7FI2mtpH1hgtwcnsh1S3Da5kwmdwDXmVk3M1vQTPM8gKQ8SSYpq+az5jquhfNem+x8XGLabaKW1B14AfgN0BsYBPwU2B9lXEkaCqw1sz1RB+JcCjvPzLoB44DxwA8jiGEosCSC5bo01G4TNTAawMyeNrMqM9tnZtPNbDGApCslvVUzsqQzJC0Pa98PSnq9pgk6HPdtSXdLKpG0WtIJ4ecFkopim4Ul9ZD0pKRiSesk/ZekjDqWO0XSsnC59wOKtzKSrgYeAY4Paws/rT2vcDyTNDJ8/bikByS9KGmXpPckjYgZ93BJM8LWhi2SfiTpLOBHwCXhchaF437SJC8pI1yndeG6PympRzis5iz/CknrJW2V9OM61qmvpBfCbbpd0ps126nWeAq3fVG4nRZLOiJmlF71rOO94Xe0U9J8SSfFDLtN0nOSng2n/UDS2JjhAyX9Nfwe10i6Pt56uNRkZpuBVwgSNgCSsiXdEe6bWyQ9JKlzQ/NSrUtS8cpezPx3A5nAIkmrws8/KZfh+08u2SisyUu6OdzHN0m6KmbczpLuDMtbqaS3wpjfCEcpCcvq8XGOLydIej+c7n1JJ9Rap58pOLbtkjRdUt861v8/JW0Ix1suaXId450j6eNwvA2SflBreF3r+AVJC8JyWqCYVreYY8o1kjaG094cMzxD0i2SVknaJmmqpN7x4ktV7TlR5wNVkp6QdLakXnWNGO6czxGcefcBlgMn1BrtWGBxOPwp4BngGGAkcDlwvz5tYvsN0AMYDpwMfB24qtb8apb7V+C/gL7AKuDEeDGa2R+AbwHvhs1pP2loA4QuI2hJ6AWsBH4eLjsHmAm8DAwM1+NVM3sZ+AXwbLicsXHmeWX4d2q4jt2A+2uNMwk4BJgM3CrpsHA9bjOz28JxbgYKgX7AAIIThHj3vD0D+DzByVdP4BJgW0PrGHqf4EDdm+B7+4ukTjHDLwD+EjP8eUkdwhOGfwKLCFpjJgM3SDozTnwuBUkaDJxNsE/U+D+C/WgcwT4/CLi1uZZpZvvD2jzAWDMbUe8EnzqI4JgxCLgaeCDmmHUHcDTBMak38P+AaoIyAdAzLKvvxs4wTFYvAvcRHLfuAl6U1CdmtK8QHJv6Ax2BH4TrsdbM8sL5HAJcBxxjZjnAmcDaOtbjD8C14XhHAK8luI57CI6TPYEvAN+WdGGteZ8KjCI4Htwi6fTw8+uBCwmOtQOBHcADdcSXktptojaznQTJwoDfA8WSpkkaEGf0c4AlZvY3M6sk2LE31xpnjZk9ZmZVwLNALnB7WDCnA+XASEmZBInkh2a2y8zWAncCX6tjuR+b2XNmVgHcE2e5yfqbmc0N1+vPfFq7OBfYbGZ3mllZGOt7Cc7zq8BdZrbazHYTnOBcqphrZcBPw1aMRQTJLl7CrwAOBoaaWYWZvWnxb05fAeQAhwIys6VmtimBdcTM/mRm28ys0szuBLIJTiBqzI/Z/ncBnYDjCE7C+pnZ7WZWbmarCfajSxPcRi46z0vaBRQARcBPIGiZAf4duNHMtpvZLoKT0lT4TisIjicVZvYSsBs4JDxh/AbwfTPbELYOvmNmiVzC+wKwwsz+GO7/TwPLgPNixnnMzPLNbB8wlZiyE6OKoNyMkdQhTOKr6lmPMZK6m9kOM/ugoXUEMLPZZvahmVWHrZ5PEyTeWD81sz1m9iHwGMEJOsC1wI/NrDDcLrcBF9c6HqW0dpuoAcID+pVmNpjg7G4gQTKsbSBBoa6ZzghqerG2xLzeF45X+7NuBDXjjsC6mGHrCM4iE1luQZzxkhGb+PeGMUJwolFXYWvIQD67flkEteKGlhvr1wS1nekKLifcEm9hZvYaQY39AWCLpIcV9EFocFlhU9vSsOmvhOCMPrZ5L3b7VxN87wMJrjEOVNAsXxJO+6Na6+hS04Vhje4UgpO7mu+7H9AFmB/znb4cfh61beGJZo2a/bgvwcljU8pq7XIKnz0WNVhOzWwlcANBAiyS9IykgXUs898IKiDrFFw+PD5mWF3riKRjJc1ScJmplKD1sHYzfOyxcV24fhCU1b/HfKdLCU4u0qastutEHcvMlgGPEyTs2jYBg2vehGfeg+OMl4itBGeOQ2M+GwJsqGO5ubWWmxtnvLrsITjw1Ex/UCOmLQDqapZr6JFrG/ns+lVy4MlMg8Ja/M1mNpzgLP+muq59mdl9ZnY0cDhB0+V/NDR/Bdej/xP4MtDLzHoCpRzYDyB2+2cQfO8bCbbPGjPrGfOXY2bnNGYdXXTM7HWCMn9H+NFWghPqw2O+0x4xTdX1OaCsETTjNsbeJk6/FSgjflltbDmFuo9F9TKzp8xsUjg/I7iEEG+8983sAoKm9OcJaumJeAqYBuSaWQ/gIT7bXyf22DiEYP0gKKtn1yqrncys0esZlXabqCUdGtamBofvcwmaSubEGf1F4EhJF4bNJd+l8QURgLBpfCrwc0k5koYCNwF/ijP6i8Dhki4Kl3t9I5e7KJx+XHjd9bZGTPsCcJCkGxR0gMmRdGw4bAuQpzgdu0JPAzdKGhZel6+5pl1Zx/hxSTpX0sjwBGUnwVlwVZzxjgnPuDsQHDDL4o0XRw7BCUQxkCXpVqB7rXGOjtn+NxD8KmAOMBfYGXai6SwpU9IRCn/e59LGPcAUSePCFpPfA3dL6g8gaVDtfgeSOtX6E7AQuEhSFwWdwq5uZBwLga+E+9FZfLZZN64w5keBuxR0bsxU0Gksm2C/riboJxLPS8BoSV+RlCXpEmAMQdlPmKRDJJ0WLrOM4GQnXjntqOB33D3CS0k7441Xhxxgu5mVSZpIcO28tv8Ot//hBNfVnw0/f4jgeDs0jKOfpAsas45Ra7eJGthF0AHsPUl7CA6+HxF0YDqAmW0FvgT8iqCT0hhgHk3/Kdf3CBLKauAtgrPFR+tZ7i/D5Y4C3k50IWaWD9xO0ClsRbisRKfdRfCb7PMImr9WEHTWgKBzFcA2SR/EmfxR4I8EvU7XEBTe7yW67Bijwth3A+8CD5rZ7DjjdSc4wO4gaPLaxqe1pPq8AvyLoGPhujDO2pcW/kHQp2AHQT+Ci8JraFUE22YcwTpuJeh13yPx1XNRM7Ni4Engv8OP/pPgcsscSTsJ9r/YPguDCBJR7N8I4G6CfihbgCcI+kI0xvcJ9qcSgj4ezzdi2h8AHxJ0jNxOUJvNMLO9BB0n3w6bfY+LncjMthH0RbmZoMz8P+Dc8LjTGNkEx6itBMeK/gSXgeL5GrA23LbfIuhom4jvALeHfQtuJX5N/HWC7+5V4I6wbxDAvQS18enh9HMIjv1pQ/H75rj6hDXJQuCrZjYr6nhcy1DwE5CRZpbowcQ518ok5RGcLHdobKtdumjPNepGkXSmpJ5h886PCK6PxGsmd84555qNJ+rEHU/Qs3IrQRPVheFPFpxzzrkW403fzjnnXArzGrVzzjmXwjxRO+eccyks5W6h1rdvX8vLy4s6DOdS3vz587eaWSrcNatOXp6dS0x95TnlEnVeXh7z5s2LOgznUp6k2rd/TDlenp1LTH3l2Zu+nXPOuRTmido555xLYZ6onXPOuRSWVKKW9KikIkkf1TFcku6TtFLSYkmfS2Z5zrmWF69cS+otaYakFeH/XlHG6Fx7kmyN+nHgrHqGn03wYIVRwDXAb5NcnnOu5T3OZ8v1LcCrZjaK4KEHcZ8N7pxrfkklajN7g+BpLXW5AHjSAnOAnpIOTmaZzrmWVUe5voDgqVCE/y9s1aCcS1OVVdVJz6Olf541iAMfG1gYfraphZfr3GeUV1azbc9+duypYGdZBaX7Kti5r4J9FVXsLQ/+yiqqKK+sZn9lFfsrq6moMiqrgv9V1dVUVhvVZlRWGWZQZcH7agMLX5vxyXsAMzBiX386rEbtG/n265bNs9ce3wpbJWEDzGwTgJltqnles3Oubjv2lHPub97ix184jHOObHodtaUTteJ89pmbi0u6hqBpnCFDhrRwSK4tK9pVxtJNu1i7dQ9rtu5h7bY9bCopo2hXGTv2VjQ4fXZWRvDXIZOOmRl0zMogK0NkZQb/MzNEVobIyBAZGdAhI4MMiQwJifA1QPBeEP4P3hPzvnbpiH3bq0vHZtoirc/Ls3OBh99czcbSfQzv1zWp+bR0oi4EcmPeDwY21h7JzB4GHgaYMGGCPyXEJWxl0W5mLy9iwfoSFhaUsKHk0weademYSV6frgzp04UJeb3on9OJfjnZ9O7age6dOtC9c/C/S3YmXTpm0ikrk4yMeOeWDtgi6eCwNn0wUFTXiF6enYOtu/fz+NtrOfeogRx6UPek5tXSiXoacJ2kZ4BjgdKa5jPnmurjjTt56cNNvLxkMyuLdgMwqGdnxg3pyVUn5nHEoB4M79uVfjnZSJ54m8k04Argl+H/f0QbjnOp7aHZq9hfWcUNp49Kel5JJWpJTwOnAH0lFQI/AToAmNlDwEvAOcBKYC9wVTLLc+2XmfH2ym08OHsl76zaRobg2GF9+NpxQ5kyZgADe3aOOsQ2o45y/UtgqqSrgfXAl6KL0LnUtmVnGX+cs44vjh/MiH7dkp5fUonazC5rYLgB301mGc7NWlbE3TPzWVxYSv+cbH50zqFcfHQuvbum73XcVFZPuZ7cqoE4l6YemLWSqmrj+5OTr01DCj6Uw7kau8oquP2fH/OX+YUM7dOFX3zxSC763CA6dciMOjTnnItrQ8k+nplbwJcm5DKkT5dmmacnapeS3lu9jZv/soiNJfu47tSRXD95FB2z/I63zrnUdv9rKwD43mkjm22enqhdynlw9kp+/cpyhvTuwl++dTxHD+0ddUjOOdegddv2MHVeIZcfO6RZ+814onYp5TevruDOGfmcN3Ygv7zoSLpm+y7qnEsP9766gqwM8d1Tm682DZ6oXQq5/7UgSV80fhC//tJYMv03zc65NLGyaDfPL9jA1ZOG0b97p2adt1/0cynhgVkruWN6PheOG+hJ2jmXdu6ZmU+nDpl86+QRzT5vT9QuclPnFfDrV5ZzwbiB3PnlcZ6knXNpZemmnbyweBNXnZhHn27ZzT5/T9QuUuu27eG2aUs4fngf7vSatHMuDd09I5+c7Cz+/aThLTJ/T9QuMpVV1dz47EIyM8SdXx5LVqbvjs659PJhYSnTP97CN08aTs8WepiOdyZzkfnt7FV8sL6Eey8d57cAdc6lpTtnLKdnlw58Y1Jeiy3DqzAuEosLS7j31RWcP3YgF4wbFHU4zjnXaPPXbWf28mKu/fwIcjp1aLHleKJ2ra6sooobnl1Iv5xsfnbBEVGH45xzTXLn9Hz6dO3IFScMbdHleKJ2re6ZuetZXbyHX/7bUfTo0nJnoc4511LeWbWVd1Zt49unjKBLx5a9iuyJ2rWqsooqHpy9imOH9ebk0f2iDsc55xrNzLhrej4Dumdz+XEtW5sGT9SulT313nqKdu3nximjow7FOeea5I0VW5m3bgfXnTqyVZ7m54natZqyiip++/oqjh/eh+OG94k6HOecazQz487pyxnUszNfPia3VZbpidq1mj+/t57iXfu54fTmeZi6c861tplLi1hcWMr1k0eSndXytWnwRO1ayb7yKn47exUnjOjDsV6bds6loepq464Z+Qzt04WLPje41Zbridq1ij+/t46tu/3atHMuff3ro80s3bSTG04fRYdWvJOiJ2rX4iqqqvndG6s5cWQfjsnrHXU4zjnXaFXVxt0z8xnZvxvnj23dmzR5onYt7o38Yop37efKE4ZFHYpzzjXJPxdtZGXRbm48fXSrPzzIE7Vrcc/NL6RP146ccoj/bto5l34qq6q5Z2Y+hx3cnbOPOKjVl++J2rWoHXvKmbl0CxeOH9Sq13Scc665/O2DDazdtpebpowmI4JH8fqR07WoaYs2UlFlXHx06/WQdM655lJeWc19r61g7OAenH5Y/0hi8ETtWtRz8ws5fGB3Dju4e9ShOOdco02dV0Dhjn3cOGU0UuvXpsETtWtByzbv5MMNpV6bds6lpbKKKu5/bSUThvaK9NkEnqhdi/nr/EKyMsT5YwdGHYpzzjXaU++tZ/POMm46I7raNCSZqCWdJWm5pJWSbokzfIikWZIWSFos6ZxklufSR0VVNX9fsJHTDu1Pn27ZUYfjmomkGyUtkfSRpKcldYo6Judawt7ySh6cvZITRvThhBF9I42lyYlaUibwAHA2MAa4TNKYWqP9FzDVzMYDlwIPNnV5Lr28kV/M1t37vdm7DZE0CLgemGBmRwCZBOXauTbnyXfXsXV3OTefEf3dFJOpUU8EVprZajMrB54BLqg1jgE1vYh6ABuTWJ5LI39fsIHeXTtyyiHR9JJ0LSYL6CwpC+iCl2nXBu0qq+B3r6/i5NH9OHpo9HdTTCZRDwIKYt4Xhp/Fug24XFIh8BLwvSSW59JERVU1ry8vZsphA+iY5d0g2goz2wDcAawHNgGlZjY92qica36Pvb2WHXsruClFnk2QzFE03pV1q/X+MuBxMxsMnAP8UdJnlinpGknzJM0rLi5OIiSXCuat3cGu/ZWceqjfiawtkdSLoNVsGDAQ6Crp8jjjeXl2aat0bwW/f3M1U8YMYGxuz6jDAZJL1IVA7FOzB/PZZrCrgakAZvYu0An4zFV5M3vYzCaY2YR+/fzgnu5mLS+iQ6aYNMq/yzbmdGCNmRWbWQXwN+CE2iN5eXbp7PdvrmZXWWXK1KYhuUT9PjBK0jBJHQk6lUyrNc56YDKApMMIErWfYrdxry0rYuKw3nTLzoo6FNe81gPHSeqi4Lcqk4GlEcfkXLPZvqecx95ewxeOOjilbtLU5ERtZpXAdcArBIV1qpktkXS7pPPD0W4G/l3SIuBp4Eozq9087tqQgu17WVm0m1O9E1mbY2bvAc8BHwAfEhw/Ho40KOea0e9eX8W+iipuPH1U1KEcIKkqj5m9RNBJLPazW2NefwycmMwyXHqZtbwIgNMO9UTdFpnZT4CfRB2Hc82taFcZT7y7lgvHDWJk/5yowzmAd8l1zeq1ZUXk9enC8H7dog7FOecS9uCsVVRUGddPTq3aNHiids1oX3kV767axqlem3bOpZGNJft46r31XPy5weT17Rp1OJ/hido1m3dWbWV/ZbU3ezvn0sr9s1ZiGN+bPDLqUOLyRO2azWvLiujSMZOJw6K/k49zziWiYPtepr5fwKXHDGFwry5RhxOXJ2rXLMyMWcuKmDSyL9lZmVGH45xzCbn31RVkZojrTkvN2jR4onbNZPmWXWwsLfNmb+dc2lhdvJu/fVDI5ccNZUD31H0QnCdq1yzeyA/uY+MP4XDOpYt7Zq4gOyuTb58yIupQ6uWJ2jWL91ZvZ3i/rhzUI3XPSp1zrsbyzbv45+KNXHliHn27ZUcdTr08UbukVVcb76/dzsQ870TmnEsP98zMp1vHLK79/PCoQ2mQJ2qXtOVbdrGzrNJ7ezvn0sJHG0r510eb+cakYfTs0jHqcBrkidolbe6a7QCeqJ1zaeHuGfn06NyBq08aFnUoCfFE7ZI2d812BvXsnLK/QXTOuRoL1u/g1WVFXPP54XTv1CHqcBLiidolxcx4b812r00759LCXTPy6dO1I1eekBd1KAnzRO2SsmbrHrbu3u+J2jmX8uau2c6bK7by7VNG0DU7qYdHtipP1C4pfn3aOZcOzIw7pi+nf042lx83NOpwGsUTtUvK3DXb6dutI8NT8IkzzjlX4+2V25i7ZjvfPXUknTqk122OPVG7pNRcn5YUdSjOOReXmXHnjOUM7NGJSyfmRh1Oo3midk22oWQfG0r2+Y1OnHMpbdbyIhasL+F7k0el5UODPFG7Jnv/k+vTfSKOxDnn4jMz7pyez5DeXbj46MFRh9Mknqhdk723ZjvdO2VxyEE5UYfinHNxvbJkM0s27uT7k0fRITM9U156Ru1Swtw12zgmrzeZGX592jmXeqqqjbtm5DO8X1cuHD8o6nCazBO1a5Ktu/ezqniP/yzLOZeyXli8kfwtu7nh9NFpXaHwRO2a5IN1OwCYkNcr4kicc+6zKququXfmCg4ZkMO5Rx4cdThJ8UTtmmRhQQlZGeLwgT2iDsU55z7j7ws2sHrrHm6cMpqMNK5Ngydq10QLC0o47ODuaXfjAOdc21dRVc19r63giEHdOfPwAVGHkzRP1K7RqqqNxYWljMvtGXUozjn3GX+ZV0jB9n3cNGV0m7gZkydq12irineze3+lJ2rnXMopq6jiN6+tYPyQnpx6SP+ow2kWSSVqSWdJWi5ppaRb6hjny5I+lrRE0lPJLM+lhoXrSwAYN8QTdXsjqaek5yQtk7RU0vFRx+RcrGfmrmdTaRk/OOOQNlGbBmjyc74kZQIPAFOAQuB9SdPM7OOYcUYBPwRONLMdktrG6U07t6CghO6dshjWxx/E0Q7dC7xsZhdL6gh0iTog52rsK6/i/lmrOHZYb04Y0XbumJhMjXoisNLMVptZOfAMcEGtcf4deMDMdgCYWVESy3MpYmFBCWNze6Z9T0rXOJK6A58H/gBgZuVmVhJtVM596o9z1rJ1935ubkO1aUguUQ8CCmLeF4afxRoNjJb0tqQ5ks5KYnkuBewtr2T55p2M9+vT7dFwoBh4TNICSY9I8mYVlxJ276/koddXc9Kovm3uRkzJJOp4pytW630WMAo4BbgMeETSZ47wkq6RNE/SvOLi4iRCci3tw8JSqs2vT7dTWcDngN+a2XhgD/CZvilenl0UHn97Ddv3lHPzGYdEHUqzSyZRFwKxD/YcDGyMM84/zKzCzNYAywkS9wHM7GEzm2BmE/r165dESK6lLSoMWjrHDvZE3Q4VAoVm9l74/jmCxH0AL8+utZXuq+DhN1Yz+dD+bfLXKMkk6veBUZKGhZ1KLgWm1RrneeBUAEl9CZrCVyexTBexhQUl5PbuTJ9u2VGH4lqZmW0GCiTVVFkmAx/XM4lzreIPb65mZ1klN50xOupQWkSTe32bWaWk64BXgEzgUTNbIul2YJ6ZTQuHnSHpY6AK+A8z29YcgbtoLFxfwtF5bev6j2uU7wF/Dk/OVwNXRRyPa+d27Cnn0bfXcvYRB7XZWxo3OVEDmNlLwEu1Prs15rUBN4V/Ls0V7SxjY2kZV7fBpiWXGDNbCEyIOg7navzujdXsKa/kxiltszYNfmcy1wgLCsIbnXiids6lgOJd+3ninbWcP3YgowfkRB1Oi/FE7RK2sKCEDpni8IHdow7FOef47exVlFdV8/3Jn+mj3KZ4onYJW7jen5jlnEsNm0r38af31nHR+EEM79ct6nBalCdql5CqauPDDf7ELOdcanhg1kqqq43r23htGjxRuwSt2Ro8Meso//20cy5iBdv38uz7BVxyTC65vdv+7eY9UbuELCooBWDs4Lb58wfnXPr4zWsrkMR1p42MOpRW4YnaJWRxYQldO2a2+WtBzrnUtnbrHv76wQa+MnEIB/foHHU4rcITtUvIosJSjhjUg0x/YpZzLkL3vrqCDpniO6eOiDqUVuOJ2jWovLKajzft5Chv9nbORWjFll08v3ADVxyfR/+cTlGH02o8UbsG5W/ZRXlltXckc85F6p6ZK+jSIZNrT24/tWnwRO0SsLiwpiOZJ2rnXDSWbCzlxQ838Y1Jw+jdtWPU4bQqT9SuQYsLS+jZpQO5vdtHxw3nXOq5e8YKcjpl8c1Jw6MOpdV5onYNWlRYypGDeiB5RzLnXOtbVFDCzKVbuOak4fTo0iHqcFqdJ2pXr7KKKvK37PJmb+dcZO6ckU+vLh24atKwqEOJhCdqV68lG3dSVW0c6T2+nXMReH/tdt7IL+bak0fQLTupJzOnLU/Url6LC4NHW3qN2jkXhTunL6dvt2y+fvzQqEOJjCdqV6/FhaX0z8nmoB7t5zeLzrnU8M7KrcxZvZ3vnDKCLh3bZ20aPFG7BiwuLPHfTzvnWp2ZceeMfA7q3omvHDsk6nAi5Yna1WlXWQWrt+7xO5I551rd7Pxi5q/bwXdPG0mnDplRhxMpT9SuTh9uKMUMT9TOuVZlZtw9I5/BvTpzyYTcqMOJnCdqV6eaO5J507dzrjXN+HgLiwtLuX7yKDpmeZryLeDq9GFhKYN7dW53t+tzzkWnutq4a0Y+w/p25aLxg6IOJyV4onZ1WlhQwthcr00751rPSx9tYtnmXdxw+iiyMj1FgSdqV4eiXWVsKNnHeE/UzrlWUlUdXJse1b8b5x41MOpwUoYnahfXwvXBjU7GeaJ2zrWSaYs2sKp4DzdNGU1mhj9boIYnahfXwoISsjLEEYO8x7dzruVVVFVz78wVjDm4O2ceflDU4aQUT9QuroUFJRx6cE67//2ic651/O2DQtZu28tNU0aT4bXpAySVqCWdJWm5pJWSbqlnvIslmaQJySzPtY7qamNxYak3e7u4JGVKWiDphahjcW3D/soq7nt1JWNzezL5sP5Rh5NympyoJWUCDwBnA2OAyySNiTNeDnA98F5Tl+Va16ri3ezeX8m43F5Rh+JS0/eBpVEH4dqOqe8XsKFkHzdPGe3PvRkzFE0AACAASURBVI8jmRr1RGClma02s3LgGeCCOOP9DPgVUJbEslwrWlDgHclcfJIGA18AHok6Ftc2lFVUcf+slUzM681Jo/pGHU5KSiZRDwIKYt4Xhp99QtJ4INfMvIksjSwsKCGnUxbD+3aNOhSXeu4B/h9QHXUgrm3405x1bNm5n5vO8Np0XZJJ1PG2qH0yUMoA7gZubnBG0jWS5kmaV1xcnERIrjksXF/CuNye3qHDHUDSuUCRmc1vYDwvzy4he/ZX8tDrqzhxZB+OG94n6nBSVjKJuhCIvVv6YGBjzPsc4AhgtqS1wHHAtHgdyszsYTObYGYT+vXrl0RILln7yqtYvmWXN3u7eE4Ezg/L8zPAaZL+VHskL88uUU+8u5atu8u5acohUYeS0pJJ1O8DoyQNk9QRuBSYVjPQzErNrK+Z5ZlZHjAHON/M5iUVsWtRH24oparaPFG7zzCzH5rZ4LA8Xwq8ZmaXRxyWS1O7yip4+I3VnHpIP44e6h1X69PkRG1mlcB1wCsEPUCnmtkSSbdLOr+5AnSta2HBDsA7kjnnWtajb62lZG+F16YTkJXMxGb2EvBSrc9urWPcU5JZlmsdCwtKyO3dmT7dsqMOxaUwM5sNzI44DJemSvaW88ibqznz8AEc6c+7b5DfmcwdIOhI5s1QzrmW8/s3V7O7vJIbp4yOOpS04InafaJoZxkbS8sY62e4zrkWsm33fh57ey3nHjWQQw/qHnU4acETtftEzY1Oxg/x69POuZbx0OurKKuo4obTR0UdStrwRO0+sWB9CR0yxeEDvUbtnGt+W3aW8eS76/ji+MGM6Nct6nDShidq94m5a7Zx1OCe/sQs51yLeHDWSqqqje9P9tp0Y3iidkBwo5PFhaVMHNY76lCcc23QhpJ9PD23gC9NGMyQPl2iDieteKJ2ACxYv4PKavNE7ZxrEfe/tgKA607z2nRjeaJ2ALy3ZjsZwu8Q5Jxrduu27WHqvEIum5jLoJ6dow4n7XiidgC8v3Y7YwZ2p3unDlGH4pxrY+59dQVZGeK7p46MOpS05InaUV5ZzQfrdzAxz59e45xrXiuLdvP8gg18/fih9O/eKepw0pInaseHG0opq6hm4jBv9nbONa97ZubTqUMm3zp5RNShpC1P1I65a7YDcEyedyRzzjWfZZt38uKHm7jyhDx/fkASPFE75q7Zxsj+3bwgOeea1d0z8unWMYtrPj886lDSmifqdq6q2pi3dof/LMs516w+LCzllSVb+OZJw+nZpWPU4aQ1T9Tt3NJNO9m1v5JjPVE755rRXTOW07NLB74xKS/qUNKeJ+p2zq9PO+ea2/x1O5i1vJhrPj+cHP/JZ9I8Ubdzc9dsJ7d3Zwb6TQicc83krhnL6dutI1eekBd1KG2CJ+p2zMyYu3a7/37aOdds5qzextsrt/Gtk0fQpWNW1OG0CZ6o27FVxbvZvqfcr08755qFmXHX9HwGdM/m8uOGRh1Om+GJuh17PX8rAMeP8Bq1cy55b63cyty127nu1JH+uNxm5Im6HZu1rIiR/buR29sfOeecS46Zccf0fAb17MyXj8mNOpw2xRN1O7V7fyXvrdnGaYf2jzoU51wb8NqyIhYVlHD95JFkZ3ltujl5om6n3lqxlYoq49RDPFE755JTXW3cOT2foX26cNHnBkcdTpvjibqdmrWsiJzsLCbk+YM4nHPJeWXJZj7etJPvTx5Fh0xPK83Nt2g7ZGbMWl7ESaP7eqFyziWlqtq4e2Y+I/p15YJxg6IOp03yo3Q7tGTjTop27fdmb+dc0l5YvJH8Lbu5acohZGYo6nDaJE/U7dCsZUUAnOKJ2jmXhMqqau6ZuYJDD8rh7CMOijqcNiupRC3pLEnLJa2UdEuc4TdJ+ljSYkmvSvJfwKeA15YXMXZwD/rl+GMtXeIk5UqaJWmppCWSvh91TC5af1uwgTVb93DTlNFkeG26xTQ5UUvKBB4AzgbGAJdJGlNrtAXABDM7CngO+FVTl+eax7bd+1lYUMKp/rMs13iVwM1mdhhwHPDdOGXetRPlldXc9+oKjhrcgyljBkQdTpuWTI16IrDSzFabWTnwDHBB7AhmNsvM9oZv5wDebz9ir+cXY4b/fto1mpltMrMPwte7gKWA9x5qp/4yv4DCHfu4acpoJK9Nt6RkEvUgoCDmfSH1F9qrgX8lsTzXDF5bVkTfbtkcMbBH1KG4NCYpDxgPvBdtJC4KZRVV/ObVlUwY2ouTR/eLOpw2L5lEHe8UyuKOKF0OTAB+XcfwayTNkzSvuLg4iZBcfcorq3kjv5hTDunn15Nck0nqBvwVuMHMdsYZ7uW5jXt67no27yzz2nQrSSZRFwKxN3QdDGysPZKk04EfA+eb2f54MzKzh81sgplN6NfPz85ayuzlRewsq/Tema7JJHUgSNJ/NrO/xRvHy3Pbtq+8igdmreL44X04YWTfqMNpF5JJ1O8DoyQNk9QRuBSYFjuCpPHA7wiSdFESy3LN4Ln5hfTtlu1NVa5JFFSd/gAsNbO7oo7HRePJd9eydfd+bj5jdNShtBtNTtRmVglcB7xC0KlkqpktkXS7pPPD0X4NdAP+ImmhpGl1zM61sG279/PasiIu+twgsvxuZK5pTgS+BpwWlueFks6JOijXenbvr+Sh11dx8uh+TMjz59i3lqxkJjazl4CXan12a8zr05OZv2s+/1i4kcpq49/8hvmuiczsLeL3TXHtxGNvrWHH3gpumuK16dbkVat24rn5hRw1uAeHHJQTdSjOuTRUureCh99czemHDWBsbs+ow2lXPFG3A0s2lvLxpp1cfLTXpp1zTfPIW6vZVVbptekIeKJuB/46fwMdMsV5Rw2MOhTnXBravqecR99awxeOOpgxA7tHHU6744m6jSuvrOb5hRs4/bAB9OraMepwnHNp6Hevr2JfRRU3nj4q6lDaJU/Ubdzs5UVs31Puzd7OuSYp2lXGE++u5YJxgxjZ3/u4RMETdRs3dV7w2+nP+2+nnXNN8NvZq6ioMr4/2WvTUfFE3YYt3bSTmUu38JWJuXTw30475xppY8k+/jxnPRd/bjB5fbtGHU675UfvNuzemSvIyc7i6knDow7FOZeG7p+1EsP43uSRUYfSrnmibqOWbCzl5SWbuWrSMHp06RB1OM65NFOwfS9T3y/gkmNyGdyrS9ThtGueqNuo+15dQU6nLK6eNCzqUJxzaei+V1eQkSGuO9WvTUfNE3UbtGRjKa8s2cLVk4bRo7PXpp1zjbO6eDd/W7CBy48dykE9OkUdTrvniboNumdmUJu+6kSvTTvnGu/eV1fQMTODb58yIupQHJ6o25yPNpQy4+MtfHPScK9NO+caLX/LLqYt2sgVJ+TRLyc76nAcnqjblMqqam79x0f06NyBqyblRR2Ocy4N3T0jn64ds7j28/5rkVThiboNeXD2Kj5YX8LPLjyC7p28Nu2ca5wlG0v510eb+cakYX7L4RTiibqNWFRQwr2vruCCcQM5f6w/fMM513h3z8inu/9aJOV4om4D9pVXcePUhfTPyeb284+IOhznXBpasH4HM5cWce3JI7x/S4rJijoAl7xfvLSU1cV7eOqbx/rNTZxzTXLXjHx6d+3IlSfkRR2Kq8Vr1Gnu+QUb+OOcdXxz0jBOGNk36nCcc2lo7prtvLliK986eThds73+lmo8UaexfyzcwE1TF3LssN784MxDog7HOZeGzIw7py+nX042XzsuL+pwXByeqNPUPxdt5MZnFzIhrzePXXUMnTpkRh2Scy4NvbNqG++t2c53TxlB545+HElFnqjT0AuLN3LDswuZMLQ3j115DF06elOVc67xzIw7pi9nYI9OXHbskKjDcXXwI3waKa+s5u6Z+fzu9VVBkr7qGL+e5JxrstnLi1mwvoRffPFIsrO8Np2q/CifJpZv3sUNzy5k6aadXHpMLreeN8Zr0s65JjMz7pyxnNzenfnShMFRh+Pq4Uf6FLd7fyVPvLOWe2euoHvnLB75+gROHzMg6rCcc2nulSVb+GjDTu740lg6ZPpV0FTmiTpF7dhTzmPvrOWJd9ZSuq+CMw8fwC++eCR9uvlN8p1zyamuNu6ekc/wvl25cJzfyTDVeaJOIfsrq3h75VZe/mgz/1y0iX0VVZwxZgDfOXUk43J7Rh2ec66NeGruepZv2cV9l40ny2vTKS+pRC3pLOBeIBN4xMx+WWt4NvAkcDSwDbjEzNYms8y2pLKqmmWbd7GwoIQ5q7cxe3kxu/dXkpOdxblHHcw1nx/OqAE5UYfp3CcaKvMu9U19v4D//sdHTBrZl3OPPDjqcFwCmpyoJWUCDwBTgELgfUnTzOzjmNGuBnaY2UhJlwL/B1ySTMDppqra2LZ7P0W79lOwfS+rt+5h7dY9rN66hyUbSymrqAagb7dszht7MGcefhAnjOhLxyw/y3WpJcEy71LYH+es47+f/4iTR/fjd187mowMRR2SS0AyNeqJwEozWw0g6RngAiC20F4A3Ba+fg64X5LMzJq60KpqY/32vUDQaxGg9sw+nbthFgwP/ofva72uNvvkr6o6WEbwOvirqKqmMvxfUWWUV1azv7KK8spq9pZXsa+iir3llezdX0Xpvgp2llWwc18l2/aUs33PfqprBdi3WzbD+nbhsolDGJfbk/G5vcjt3RnJC41LaYmU+UbZuns/u8sqmyk8V58ZH2/h5y8t5fTD+vPAVz/nP8dKI8kk6kFAQcz7QuDYusYxs0pJpUAfYGtTF1q6r4JT75jd1MlbRKcOGXTpmEXnDpn06NyB7p2zyOvbhfFDetI/J5t+Odn0y+nE4F6dyevblW7+22eXnhIp841yz8x8/jRnfVJBucSdfcRB3HvpeG+xSzPJZIx41b/aldtExkHSNcA1AEOG1H93nK7ZmdxzybiYaesILhygcByh8H/4Xgpfi8yM4H+GRKZERgZkSmRmBH8dMjPIyhRZGaJjZibZHTLomJlBx6wMOnfI9OYj1140e3m++Ohcjh7aq1mCc/Xr0jGL0w7t7z/FSkPJJOpCIDfm/WBgYx3jFErKAnoA22vPyMweBh4GmDBhQr3N4tlZmVw4flASYTvnmiiRMt+o8jwut6f/osG5BiRzavU+MErSMEkdgUuBabXGmQZcEb6+GHgtmevTzrlIJVLmnXPNrMk16vCa83XAKwQ/1XjUzJZIuh2YZ2bTgD8Af5S0kqAmfWlzBO2ca311lfmIw3KuzUuqV5OZvQS8VOuzW2NelwFfSmYZzrnUEa/MO+dalvcqcM4551KYJ2rnnHMuhXmids4551KYJ2rnnHMuhSnVfi0lqRhYl8CofUniDmcR8rhbV1uOe6iZ9WuNYJoqwfLclr+jVJSucUP6xp5UeU65RJ0oSfPMbELUcTSWx926PO7Ul67r6nG3vnSNPdm4venbOeecS2GeqJ1zzrkUls6J+uGoA2gij7t1edypL13X1eNufekae1Jxp+01auecc649SOcatXPOOdfmpVWilvQlSUskVUuaUGvYDyWtlLRc0plRxZgISbdJ2iBpYfh3TtQx1UXSWeE2XSnplqjjaQxJayV9GG7jeVHHUxdJj0oqkvRRzGe9Jc2QtCL83+Ye2twWynM6lWVI3/Lc3styWiVq4CPgIuCN2A8ljSF4MtfhwFnAg5IyWz+8RrnbzMaFfyn5kINwGz4AnA2MAS4Lt3U6OTXcxqn8k47HCfbbWLcAr5rZKODV8H1b01bKc8qXZWgT5bndluW0StRmttTMlscZdAHwjJntN7M1wEpgYutG1yZNBFaa2WozKweeIdjWrhmZ2RsEj4GNdQHwRPj6CeDCVg2qFXh5bnVenltYS5XltErU9RgEFMS8Lww/S2XXSVocNpWkarNmOm7XWAZMlzRf0jVRB9NIA8xsE0D4v3/E8bSmdNvv0qEsQ/pt11jtuiwn9TzqliBpJnBQnEE/NrN/1DVZnM8i7c5e33oAvwV+RhDjz4A7gW+0XnQJS7nt2kgnmtlGSf2BGZKWhWe8rpW0hfLcRsoypNh2baR2XZZTLlGb2elNmKwQyI15PxjY2DwRNU2i6yHp98ALLRxOU6Xcdm0MM9sY/i+S9HeCpr90KdxbJB1sZpskHQwURR1QU7SF8txGyjKk2HZtjPZelttK0/c04FJJ2ZKGAaOAuRHHVKfwy6rxRYJONanofWCUpGGSOhJ08JkWcUwJkdRVUk7Na+AMUnc7xzMNuCJ8fQVQV+2zLUqb8pxGZRnStDx7WU7BGnV9JH0R+A3QD3hR0kIzO9PMlkiaCnwMVALfNbOqKGNtwK8kjSNodloLXBttOPGZWaWk64BXgEzgUTNbEnFYiRoA/F0SBPv5U2b2crQhxSfpaeAUoK+kQuAnwC+BqZKuBtYDX4ouwpbRRspzWpRlSOvy3O7Lst+ZzDnnnEthbaXp2znnnGuTPFE755xzKcwTtXPOOZfCPFE755xzKcwTtXPOOZfCPFE755xzKcwTtXPOOZfCIk3Ukr4qaXqqLVfSbEnfrGOYJD0maYekVr1bkqR/Sbqi4TFbR33bKcHpd0sa3pwxhfM9SVK8pzKlFEmPS/qfqONwrU/SKeENMZo6/Y8kPdKcMcXMO6WOM/FIypNkktLqpl1N1eKJWtIkSe9IKpW0XdLbko4BMLM/m9kZLR1DbUkudxIwBRhsZi326D0FD6T/U+xnZna2mT1R1zSNnPdtyc4nWWbWzcxWJzufsMCOjJnvm2Z2SLLzjbOclNhuLiBpraR94Qnf5vDEp1uC0zY5UUqK/C5RZvYLM2vySXKNljzOxFlW5NstXbVoopbUneAm9b8BehM8Uu2nwP6WXG4LGwqsNbM9UQeSrtrLWbBrFeeZWTdgHDAe+GHE8bQ4Lz/tT0vXqEcDmNnTZlZlZvvMbLqZLQaQdKWkt2pGlnSGpOVh7ftBSa/XNK2G474t6W5JJZJWSzoh/LxAUlFsc42kHpKelFQsaZ2k/5KUUcdyp0haFi73fuI/Do7wXq2PAMeHZ/E/rT2vcLxPanjhWf4Dkl6UtEvSe5JGxIx7uKQZYWvDlrBJ6yzgR8Al4XIWheN+0tQsKSNcp3Xhuj8pqUc4rKZZ6ApJ6yVtlfTjOtapr6QXwm26XdKbNdspzrj1bidJ35C0VMFlgVckDa21Tb4raQWwInY7STourBFlxoz/RUk1+8lESe+GMW6SdL+ChwogqeYJOovCbXVJbG1J0i2SnqsV572S7gtf95D0h3C+GyT9T2wcdUl0uylwd/gdlSp4bvERMaP0qmffuDfct3cqeA7vSTHDbpP0nKRnw2k/kDQ2ZvhASX8N9/81kq5vaJ3SmZltJriH9biazxQ81OOOsAxskfSQpM4NzUu1LunEK+O1hq0Ov4M1kr5ax3idw2PBDkkfA8fUGl7n9xXzXf9J0k7gSsXUhCW9rOAe3rHzWyTpovB13P1IDRxnwu1XEru/SuqnoBWjf/j+XEkLw/HekXRUQ9u3kdttoqR5YexbJN1Va5SvKs4xrr5jRjjcJF0fxrBV0q9jy6/qOZZFwsxa7A/oDmwDngDOBnrVGn4l8Fb4ui+wE7iI4Mbr3wcqgG/GjFsJXEVwQ/n/IbjB+QNANsETVXYB3cLxnyR4SkkOkAfkA1fXs9yLgQ7AjeFyvlnHOn0ybbz34WcGjAxfPw5sJ3gsWxbwZ+CZcFgOsAm4GegUvj82HHYb8Kda850dsz2+AawEhgPdgL8BfwyH5YUx/B7oDIwlaMU4LM76/C/wULjuHYCTCO8BX2u8ercTcGEYz2Hhev4X8E6tbTKDoGWlc5zttAqYEjP+X4BbwtdHA8eF880DlgI3xNve4ftTgMLw9VBgL9A9fJ8ZbvPjwvfPA78DuhI80H0ucG0C+3ai2+1MYD7Qk+DE5jDg4Ib2jXD45UCfcNjNwGagU8z+URHzffwAWBO+zgiXeSvQMdxHVgNntmR5b+0/godgnB6+Hgx8CNwbM/wegicX9SYoW/8E/rf2PhJnvrOJKf/EKePh510JysQh4fuDgcPrmOcvgTfDWHIJnv5Us4/W+33FfNcXhuN2Jub4AHwdeDtmWWOAEiA7wf2ovuPMo8DPY4Z9F3g5fP05gkc2HktQrq4Iv5PsBr63xmy3d4Gvha+78Wm5zaOeYxyJHTNmhd/HEIL8kNCxLJJ9vRUK02EEB6RCggP7NGBA7QIQ7mzvxkwnoIADE/WKmOFHhht7QMxn2wjOqDPDL21MzLBrgdl1LHdOreUW0ryJ+pGYYecAy8LXlwEL6ljObdRfgF4FvhMz7BCCwlyzYxrBdfSa4XOBS+Ms53aCE5qR8eKIGa/e7QT8i/BEKHyfQZAgh8Zsk9Pq2U7/Q/A0HwgOqntqpo0Tyw3A3+PNJ3x/CjEHYeAt4Ovh6ynAqvD1gHA/6Rwz7mXArAT260S322kEB4HjgIxaw+rcN+qY1w5gbMz+Eft9ZBCcgJxEcOBcX2vaHwKPNaUMp+ofQVLYTXCCbmGZ6Bmzf+4BRsSMfzywJt4+Umu+n5Sz8P2V1J2oS4B/i92H6pjnauCsmPfX8Gmirvf7Cr/rN2oNv41PE/UB5QX4eU1ZSnA/qu84czqwOmbY2zFl6bfAz2pNuxw4uYFt0Zjt9gbB5dK+tT7PI8FjXDgs3jEj9vv4DvBq+LreY1kUfy3emczMlprZlWY2GDgCGEhwplvbQILEXDOdESSCWFtiXu8Lx6v9WTeC2l9HYF3MsHUE18gTWW5BnPGSsTnm9d4wRgjOrFc1cZ4D+ez6ZREkn4aWG+vXBGeP08NmoFvqWV5922kocG/Y1FRCUFMUB27z+rbrU8BFkrIJWlU+MLN1AJJGh83Mm8Omv18QfMeJeoogAQN8JXxfE3MHYFNM3L8jqFk3JKHtZmavAfcTtPxskfSwgr4bNer8jiTdHDa/lYax9eDA9Y79PqoJysvAcL0G1qxTOO2POHDfaCsuNLMcgsR7KJ9un35AF2B+zDZ4Ofy8WVjQT+US4FsE+9CLkg6tY/QDyg8Hlt1Evq86y46Z7QJeJHi+NOH/P9cMT2A/qs9rQGdJx4bNv+OAv8fEfXOtuHPDda1TI7fb1QSXUJdJel/SubWGxy0/CR4zan8fNXEncixrVa368ywzW0ZQizgizuBNBM1XQHBtL/Z9I20lqF0OjflsCLChjuXm1lpubpzx6rKH4IBQM/1BjZi2ABhRxzBrYNqNfHb9KjnwZKZBZrbLzG42s+HAecBNkibHGbWh7VRA0GTcM+avs5m9E7u4euL4mKCwnM2ByRSCM/dlwCgz605wEIvbj6AOfwFOkTQY+GLMvAsIatR9Y2LubmaHNzTDRmw3zOw+MzsaOJzgoPMfDc0/vI74n8CXCS4Z9QRKOXC9Y7+PDILysjFcrzW1voscMzunoeWmKzN7neDYckf40VaCE/fDY7ZBDws6njXkgDIN1FmmzewVM5tC0Hy7jKApNp4Dyg9Bea2RyPfV0PHgaeAySccTNAXPgoT2o3rnG54ATiU40f0K8EJ4YlAT989rxd3FzJ5uINaEt5uZrTCzywhOnv8PeE5S14bmT2LHjNrfx8aY9WroWNaqWrrX96Hh2dzg8H0uwRc+J87oLwJHSrpQQa/G71JPAamPBQ+Znwr8XFJOeCZ4E/CnOKO/CBwu6aJwudc3crmLwunHSepE0JSUqBeAgyTdEHbcyJF0bDhsC5CnOjp2ERTMGyUNU/CTlF8Az5pZZSOWX9MZZGSYeHcCVeFfbQ1tp4eAH0o6PJxvD0mNfUD6U+F8P0+QXGvkhLHtDs+8v11rui0E1/XiMrNigua8xwgOiEvDzzcB04E7JXVX0EFvhKSTGwo00e0m6ZiwNtKBIAGUxRsvjhyCE69iIEvSrQR9PmIdHfN93EBw0jGHoAlwp6T/VNCJKVPSEQp/FtmG3QNMkTQuTDC/B+7Wpx2fBkk6M3YCSZ1q/QlYSNC600VBp9Cr4y1M0gBJ54eJYz9BM3xd3+1UgvLRKzwefi9mWHN8Xy8RnLjfTnAcqA4/b2g/aug4A0G5vAT4KgeeQP8e+Fa4f0tSV0lfkJRTX6CN2W6SLpfUL1yfkvDjRMtPfccMgP8Iv49cgj5Rz4afN8exrFm1dI16F8H1l/ck7SE4iHxE0KHhAGa2FfgS8CuCa81jgHk0/adc3yM4MK4muEb5FEHHiLqW+8twuaMIrsMkxMzyCQrHTILezHF7h9Yx7S6Ca6bnETThrABODQfXJKptkj6IM/mjwB8JruGsIUgA34szXkNGhbHvJui48aCZzY4Ta73bycz+TnDG+0zY1PQRQe24MZ4maMJ8LVxejR8QnM3vIjg4PFtrutuAJ8Kmqi/XMe+nCK63PVXr868TXCb5mODa3XMEZ/kNSWi7ERwUfx/Oex3Btrsjzni1vUJwrSw/nK6MzzZ//oPgALoD+BpwkZlVhCeq5xE0U64hqF0+QtDk2WaFJ2RPAv8dfvSfBJcn5oT75EyCvhw1BhHUumP/RgB3A+UESewJYpqRa8kgOJZtJGgePZngWmc8PyX4HtcQnBz+MSbupL8vM9tP0KG09j7e0H7U0HEGM3uP4Fg6MJxXzefzgH8nuLSzg2BbX5lAuI3ZbmcBSyTtBu4luAZdlsAyGjpmQFB+5hOcmL0I/CFcr+Y4ljUrBZcaU094hlcIfNXMZkUdj3OpRMGNV0aa2eVRx+JculFw85VRZrYy6lgSkVL3+pZ0pqSeCjoU1VxTiNdM7pxzzrULKZWoCX4+sYqg6ec8gh6d+6INyTnnnItOyjZ9O+eccy71atTOOeeci+GJ2jnnnEthKfcUlr59+1peXl7UYTiX8ubPn7/VzJrtTlstwcuzc4mprzynXKLOy8tj3rx5UYfhXMqTtK7hsaLl5dm5xNRXnr3p2znnnEthnqidc865FJZQopZ0lqTlklYqzlOCJN0k6WNJiyW9qpiHQHI3lgAAIABJREFUbEuqUvBg8YWSpjVn8M4551xb12CilpRJ8Ii+swnuv32ZpDG1RlsATDCzowjulfyrmGH7zGxc+Hd+M8XtnGtGkh6VVCTpozqGS9J94cn6Ykmfa+0YnWuvEqlRTwRWmtlqMysHngEuiB3BzGaZ2d7w7Rya/nhK51w0Hid4AEJdziZ4EMko4BqCxwg651pBIol6EAc+baWQ+h+gfTUxT1gBOkmaJ2mOpAvjTSDpmnCcecXFxQmE5JxrTmb2BsGTjOpyAfCkBeYAPSUl8pQx51ySEvl5Vu2HbUMdDxuXdDkwgeCxZTWGmNlGScOB1yR9aGarDpiZ2cPAwwATJkzwe5qmiJc/2sQrS7Z88l7AFSfkMTa3Z3RBuajUdcK+KZpwXDJWFu3moddXUVXth9vWcOkxuRw7vE+Tp08kURcCuTHvBxM8R/QAkk4HfgycHD4bFQAz2xj+Xy1pNjCe4MEbLsX977+WsW13Ob27dgRgc2kZ+yureeCrfnmyHWrMCfs1BM3jDBkypCVjck308kebeG5+Ibm9O6O4X61rTmeMGZDU9Ikk6veBUZKGARuASwkeyP0JSeOB3wFnmVlRzOe9gL1mtl9SX+BEDuxo5lJUwfa9/7+9ew+O867vPf7+7q4ulmTLti52bPlu5eIQh4AJkIQ00DQkYUigTSDpaRvaTNN2SNuBHubAoQOccIahcDr0tCcUaJtCe4YmAabF0PSkCaRNSLjYkDgQh8Sy44tiY918k2RJe/meP3ZXVhTZWlurfZ7f7uc1o/Fent39Sj8/+/1dnx/7Bkf5xDs38b4r1wHwX7+2g0efP0w25yQTOrlrTEkVdlAPWQgmsvliefxDb8VM53LczTpG7e4Z4G7gYeB54EF3f87M7jGz4izuzwItwNemLcO6CNhuZjuAx4BPu/vOsv8WUnbf6xkA4Kru9snHrtrYztHRNDsPHo8qLInOVuC3CrO/3wQcc3d1ewcqnc1RlzQl6UCUdAlRd38IeGjaYx+bcvva07zuKeCSuQQo0fhezwDLFjWwoaNl8rErNrZNPndJV2tUock8MLN/Aq4B2s2sF/g4UAfg7l8gf/7fCPQAo8BvRxOplEMmmyOV0PWuQhG7a31L9HI556meAd56YecratydCxu5YNlCnuwZ4A+u2RBhhFJu7n77LM878P4KhSPzLJ116pJqTYdCVSp5lZ2HjnNkNM1bpnR7F13V3c6P9g4xls5GEJmIlEO+61tf/6FQScmrFMenr9wwQ6Le2M5EJsf2vUcqHZaIlEkm66TUog6GErW8ypM9A5y/rIXORY2veu7ydUtJJWwymYtIeNSiDotKSl5hLJ3lRy8NceXGV7emAZobUrxu9RKeVKIWCVY650rUAVFJySv8ZN8RxjM5rjpNoga4cmM7Pzt4jCMjExWMTETKJVNYniVhUKKWV/hezwCphJ3xcndXdbfjDt/fM1jByESkXNJanhUUlZS8wvd6Brhs9WJaGk6/cu/SrlZaGlI8sUvd3yIh0vKssChRy6SjoxP89OVjpx2fLkolE7xpfZvGqUUClclpMllIVFIy6fu7B3HnjOPTRVdtbGP/0Cj7B0dnPVZE4iWd0fKskChRy6SnDxylPpUoaRvL4hj20we0nlokNGm1qIOikpJJu/uGWd/eXNIJvL6jmYTB7v6RCkQmIuWUyWp5VkhUUjJpd//wKzbhOJOGVJJVS5vY3T88z1GJSLnlZ32r6zsUStQCwHgmy/6hUTZ0NJf8mg0dLezuU6IWCY2uTBYWlZQAsG9wlJzDhs7SWtQAGzqaeWlghGzO5zEyESk3Lc8KixK1AEy2jEvt+i4eO57JcfDoyfkKS0TmQSabI6UWdTBUUgIwOda8rv0sur4Lre8ejVOLBEXX+g6LSkqA/OztFa2NNJ/himTTFVvfGqcWCUta1/oOihK1AIUZ32cxPg2wtLmeJU11WqIlEphM1nWt74CopAR3Z3df6UuzptrQ0aIlWiKBSWdz1KXUog6FErVw+Pg4IxPZs1qaVbSho4U9StQiQUlnc9SpRR0MlZRMtojPqUXd2czA8ARHR7U3tUgIcjkn5+ha3wFRopZTifosx6hhyoQyjVOLBCGdywFo1ndAVFLC7r5hWhpSdC5sOOvXnkrU6v4WCUE6m79AkWZ9h0OJWtjdP8KGjmbMzv7E7VqygPpkQolaJBCZbL5FrVnf4VBJyVltxjFdKplgbXsTu/vU9S0SgolCoq5L6es/FCqpGjcynuHQsbFzGp8u2tDRwp4BtahFQpApdn1r96xglJSozex6M3vBzHrM7MMzPP9BM9tpZs+a2XfMbM2U5+4ws12FnzvKGbzM3UsD+ZbwuSzNKtrQ0cL+wVHShZq6iMTXZKLWZLJgzFpSZpYE7gVuADYBt5vZpmmHPQ1scffNwNeBzxReuxT4OPBG4HLg42a2pHzhy1zNZWlW0YbOZjI5Z9/gaLnCkgiUUCFfbWaPmdnThUr5jVHEKXNT7PrW8qxwlFKluhzocfc97j4B3A/cPPUAd3/M3Yvf0j8Augq33w484u5D7n4EeAS4vjyhSzns7hsmmTBWtzWd83to5nf4SqyQ/ynwoLtfBtwGfL6yUUo5ZLQ8KzillNRK4MCU+72Fx07nTuDfzvG1UmG7+0dYvbSJhlTynN9jvRJ1NZi1Qg44sKhwuxU4WMH4pEzU9R2eUrZKmql/xGc80Ow3gC3AL53Na83sLuAugNWrV5cQkpRLfsb3uY9PA7Q0pFi+qFEzv8M2U6X6jdOO+QTw72b2h0AzcG1lQpNyUtd3eEqpUvUCq6bc72KGmrSZXQt8FLjJ3cfP5rXu/iV33+LuWzo6OkqNXeYom3P2DIzMaXy6aENns1rUYSulUn078GV37wJuBP7RzF71HWJmd5nZdjPb3t/fPw+hylycmvWtFnUoSimpbUC3ma0zs3ryY1Nbpx5gZpcBXySfpPumPPUwcJ2ZLSlMIruu8JjEwMtHTjKRybF+ji1qgPXt+V203GfsbJH4K6VSfSfwIIC7fx9oBNqnv5Eq3vFWvOCJrkwWjlkTtbtngLvJJ9jnyU8mec7M7jGzmwqHfRZoAb5mZs+Y2dbCa4eAT5JP9tuAewqPSQzsLqx9Xl+GFvX6jmZOjGUYGNbmHIGatUIO7Ad+GcDMLiKfqNVkDsyprm+1qENRyhg17v4Q8NC0xz425fZpx6rc/T7gvnMNUObPgaH8RP01S899xnfRmsKs8f1Do3ScwzXDJVrunjGzYoU8CdxXrJAD2919K/AnwN+Y2QfId4u/z9WFEpyMrvUdnJIStVSn/YOjNNYlypJYVy/Nd58fGBrl9Wu0VD5EJVTIdwJXVjouKa90VsuzQqOSqmH7h0ZZvbTpnDbjmK5ryYLJ9xSR+Ern1KIOjRJ1DSsm6nJorEuyfFGjrk4mEnMZtaiDo5KqUe7O/qFRVpUpUQOsXto0Oe4tIvGU1mSy4KikatTgyASjE9mytagBVi1tUte3SMyltXtWcJSoa1QxoZYzUa9e2sQvjo8xls6W7T1FpLzU9R0elVSNmlyaNYfNOKYrvlfvkZNle08RKa9ii1qXEA2HEnWN2l+Y9NW1pLxd34DGqUViLK3ds4KjkqpR+4ZGWbaogca6c981a7piN/q+QW3OIRJX2j0rPCqpGlXOpVlF7S31LKhLsn9IXd8icZXO5jCDpCaTBUOJukYdKPPSLAAzY7VmfovEWjrr2jkrMCqtGjSWzvKL42Nlb1FDfpxaY9Qi8ZXO5nRVssAoUdegl4+exL28M76L1rTlW9Taq0EknjLZnC52EhiVVg0qzviejxb16qVNnExn6R8eL/t7i8jcpXOuiWSBUWnVoOIYcrnHqOFU8lf3t0g8pTPq+g6NEnUN2j9U2N6ypfz7RheTvyaUicRTJue62ElglKhrUDm3t5xucrvLQS3REomj/GQyffWHRKVVgw7MwxrqouJ2l2pRi8RTOpvT8qzAqLRqTHF7y9VLm+ftM1a3NbF/SFcnE4mjTFZd36FRoq4xA8PF7S0XzNtn6KInIvGlWd/hUWnVmMntLedhDXXR6qVNHD4+ru0uRWJIs77Do0RdYw7Mwz7U0xXfu/eIWtUicZPJ5UhpjDooKq0aU2xRl3N7y+m0REskviayTl1KX/0hUWnVmP3zsL3ldMUWdfEKaCISH5lsjjrtnBUUJeoas39wlDXzOOMb8ttdNtUn2acWtUjsZLKaTBYalVaN2T8P21tOV9zuUpcRFYmfdDan5VmBUaKuIcXtLVfN49KsolVLm9inrm+R2EnndGWy0JRUWmZ2vZm9YGY9ZvbhGZ6/2sx+YmYZM7tl2nNZM3um8LO1XIHL2Su2cNe2zW/XN8CawlrqXE7bXYrESb7rWy3qkMyaqM0sCdwL3ABsAm43s03TDtsPvA/46gxvcdLdX1v4uWmO8cocFFu487mGumhNWxPjmRx9J7TdZShmq5AXjnmPme00s+fMbKbzXWIurf2og5Mq4ZjLgR533wNgZvcDNwM7iwe4+97Cc7l5iFHKZO9g/rKeFWlRFz5j7+AIy1sb5/3zZG6mVMh/BegFtpnZVnffOeWYbuAjwJXufsTMOqOJVuYinXXN+g5MKdWqlcCBKfd7C4+VqtHMtpvZD8zsXWcVnZTV/qFRFjakWNJUN++ftaZNS7QCM1khd/cJoFghn+p3gXvd/QiAu/dVOEYpg4x2zwpOKaU1U9XrbAYeV7v7FuDXgb8wsw2v+gCzuwrJfHt/f/9ZvLWcjb2Do6xpn5/tLadbuXgBqYRNtuIl9kqpkJ8PnG9mTxYq3tdXLDopm3TW1fUdmFJKqxdYNeV+F3Cw1A9w94OFf/cA/wFcNsMxX3L3Le6+paOjo9S3lrO0f3Bk3tdQF6WSCVYuWaC11OEopUKeArqBa4Dbgb81s8WveiNVvGPL3QuzvtX1HZJSEvU2oNvM1plZPXAbUNLsbTNbYmYNhdvtwJVMGduWyslkc/QeOTnZJV0Ja9qa1fUdjlIq5L3AN9097e4vAS+QT9yvoIp3fGVzjjvq+g7MrKXl7hngbuBh4HngQXd/zszuMbObAMzsDWbWC9wKfNHMniu8/CJgu5ntAB4DPj11copUzsGjY2RyXtlEvbSJvYMjuGuJVgBKqZD/C/BWmKx4nw/sqWiUMieZwnJJXfAkLKXM+sbdHwIemvbYx6bc3ka+Bj79dU8Bl8wxRimDfUP5seI1FZjxXbSmrYkTYxmOjqZZ0lxfsc+Vs+fuGTMrVsiTwH3FCjmw3d23Fp67zsx2AlngQ+4+GF3UcrbS2fzCnHq1qINSUqKW8O0tdEFXuus7/9kjStQBKKFC7sAHCz8SoHS20KLW8qygqFpVI/YPjtCQSrBsYeXWNK9t03aXInGSKbSoNes7LCqtGrF3cJTVS5tIVLAmXdz8Y++AErVIHKQLY9Tq+g6LSqtG7B8crej4NEBjXZLzWhsnx8dFJFrpTLFFra7vkChR1wB3Z9/QSEXHp4tWaxctkdjI5NT1HSKVVg3oOzHOWDo3OWZcSWvbmpWoRWKiOJmsXi3qoChR14C9A/mu59UV7vrOf2YTA8PjDI9nKv7ZIvJKxeVZqYS++kOi0qoB+yb3oY6mRQ3anEMkDiaXZ6lFHRQl6hqwb3CEZMJYsXhBxT+7OC6+T5tziEROFzwJk0qrBuwbHGXl4gWRXN93dTFRay21SOQyky1qffWHRKVVA/YPjUYy4xtgUWMdS5vrNaFMJAbShVnf2j0rLErUNWDvQDRLs4ryS7TU9S0SteI6au2eFRaVVpU7OjrB8bHM5KSuKKxt01pqkTjQ7llhUqKucsXNOFYvjbBF3dbMwWMnGc9kI4tBRE5NJlOLOiwqrSpX7HJe2x5ti9odeo+cjCwGETm1PKtO66iDotKqcvti0KLWEi2ReDi1e5a6vkOiRF3l9g2OsnxRI411ychiKG4GonFqkWgVd89S13dYVFpVbt/gyORa5qi0NdfT0pCavJSpiETj1KxvtahDokRdxdydnv5hNnS0RBqHmbG+o5k9StQikdLuWWFSaVWxwZEJjo6m2dgZbaIG2NjRwq7Dw1GHIVLTJieTqUUdFCXqKlZMjN1xSNTLWvjF8TGOj6WjDkWkZk0uz9Ks76CotKpYT38+UcelRQ2wu0+tapGoZLJOMmEkEmpRh0SJuor1HD5Bc32S81obow6F7mULAdilRC0SmXQ2R0pJOjhK1FWsp3+YjZ0tmEV/Yq5asoD6ZEItapEIpbOupVkBUolVsV2Hh9nYuTDqMID8LNP1Hc1qUYtEKJPLaSJZgJSoq9Sxk2n6TozTvSz68emijZ0t9ChRi0Qmnc1paVaAVGJVqpgQN0a8hnqqjZ0tHDgyylham3OIRCGddeo0Rh2ckhK1mV1vZi+YWY+ZfXiG5682s5+YWcbMbpn23B1mtqvwc0e5ApczK44Fx6lF3d25EHfY3a9WtUgUMtkcdSm1z0Iza4mZWRK4F7gB2ATcbmabph22H3gf8NVpr10KfBx4I3A58HEzWzL3sGU2u/pOUJ9K0LUk2suHTlVcJqbu73iarUI+5bhbzMzNbEsl45O5S2dds74DVErV6nKgx933uPsEcD9w89QD3H2vuz8L5Ka99u3AI+4+5O5HgEeA68sQt8xiV1/+0qHJGJ2Ua9ubSCZMVyiLoRIr5JjZQuCPgB9WNkIph3Q2p1nfASqlxFYCB6bc7y08Voq5vFbmoKdvOBYXOpmqIZVkzdImtajjadYKecEngc8AY5UMTspDiTpMpZTYTE0yL/H9S3qtmd1lZtvNbHt/f3+Jby2nMzqRoffIyVhcOnS6jZ0t7Oo7EXUY8mqzVqrN7DJglbt/u5KBSflkcq69qANUSqLuBVZNud8FHCzx/Ut6rbt/yd23uPuWjo6OEt9aTmdPf36Xqri1qCEf077BUSYy00dJJGJnrFSbWQL4HPAns76RKt6xpRZ1mEopsW1At5mtM7N64DZga4nv/zBwnZktKUwiu67wmMyjYos1ji3q7mUtZHLOvkFteRkzs1WqFwKvAf7DzPYCbwK2zjShTBXv+MpfmUwt6tDMmqjdPQPcTT7BPg886O7Pmdk9ZnYTgJm9wcx6gVuBL5rZc4XXDpEf09pW+Lmn8JjMo56+YVIJY01bc9ShvEp34UppGqeOnTNWyN39mLu3u/tad18L/AC4yd23RxOunItMNkdKO2cFJ1XKQe7+EPDQtMc+NuX2NvI18Jleex9w3xxilLO06/Awa9qaqI/hesn1HfnKw66+YW6IOBY5xd0zZlaskCeB+4oVcmC7u5faiyYxpmt9h6mkRC1h6ekf5vyYXON7uqb6FF1LFqhFHUOzVcinPX5NJWKS8sqPUavrOzSqWlWZ8UyWfYOjsZxIVpSf+a1ELVJp+Vnf+toPjUqsyuwdGCWb81hdOnS67s4WdvcPk82VuspPRMpBLeowKVFXmWKX8oYYbcYx3cbOFiYyOXqPjEYdikhNSWdz1GkyWXBUYlVmV98JzOKeqPPj5y/qUqIiFZXJ6oInIVKirjLPHTzOurZmFtQnow7ltC5YvhAz2HnweNShiNSUCV3wJEgqsSrzbO9RNne1Rh3GGbU0pNjQ0cKzvUejDkWkpmR0wZMgKVFXkcPHxzh8fJzNXYujDmVWm7ta2dF7DHdNKBOplExOLeoQqcSqyI4D+Rbqpavi3aIGuLRrMQPD4xw6pk2YRCrB3fP7UStRB0clVkV++vIxkglj03nxT9SXFLrn1f0tUhmZwnLIuhjtUS+lUaKuIjt6j9Hd2RLriWRFm85bRCphPNt7LOpQRGpCJltI1DG8tLCcmUqsSrg7z/Ye5dIAxqcBGuuSXLB8oRK1SIVMZPNby6bUog6OEnWV6D1ykqOjaTYHMD5dtLlrMc/2HtWEMpEKyBQStSaThUclViV2FMZ6N68Mo0UN+Znfx8cy7B3UFcpE5tvkGLUSdXBUYlXi2d5j1CcTXLA8nrtmzWSzJpSJVMxEptD1rXXUwVGirhI7DhzlohWLYrkH9emcv2whDamExqlFKuBUi1qJOjThfKvLaeVyzs9ePsalMb8i2XR1yQQXr1ikFrVIBaQ1Rh0slVgV2DMwzMhElktWhpWoIT+h7GcvH5+c6CIi8yM9OetbX/uhUYlVgR0H8l3Hl64KZyJZ0eauVk6ms/T0ayctkflUXEddn1LXd2iUqKvAs71HaapPxnpry9MpXpdc49Qi80st6nCpxKrAsy8f4zUrW0kGeCGD9e3NLGxIaZxaZJ6lCy1qzfoOjxJ14NLZHDsPHg9uIllRImG8ZmWrWtQi8yyTy7eo6zWZLDgqscC98IsTjGdyXBLIpUNnsrmrlecPHWc8k406FJGqNdn1rUQdHJVY4LbtHQLgsgAnkhVdtnoJ6axPTooTkfKb7PoOcIis1ilRB+7xF/tZ197MqqVNUYdyzt68oY1kwnj8xf6oQxGpWqdmfetrPzQqsYCNZ7L8YM8QV3e3Rx3KnLQuqOO1qxbz+C4lapH5ktbuWcFSog7Y9r1HOJnOcvX5HVGHMmdXd3fw05ePMTQyEXUoIlVJVyYLl0osYI+/2E9d0njT+raoQ5mzq89vxx2+1zMQdSgiVak4Rq1EHZ6SSszMrjezF8ysx8w+PMPzDWb2QOH5H5rZ2sLja83spJk9U/j5QnnDr22P7xrg9WuW0NyQijqUOdvctZjWBXUap45QCef5B81sp5k9a2bfMbM1UcQp56a4PEvrqMMza6I2syRwL3ADsAm43cw2TTvsTuCIu28EPgf82ZTndrv7aws/v1+muGte34kxnj90vCq6vQGSCeOqje08sasfd486nJpT4nn+NLDF3TcDXwc+U9koZS7Uog5XKSV2OdDj7nvcfQK4H7h52jE3A18p3P468MtmpmrbPHrixXwX8dXd1ZGoId/9ffj4OC8cPhF1KLVo1vPc3R9z99HC3R8AXRWOUebg1Bi1vppDU0qiXgkcmHK/t/DYjMe4ewY4BhQHTteZ2dNm9p9m9pY5xisFj+/qp72lnk3nLYo6lLJ5S6HSUayESEWVcp5PdSfwb/MakZRVRtf6DlYpJTZT9Wt63+TpjjkErHb3y4APAl81s1dlFjO7y8y2m9n2/n6NUc4ml3Oe2DXAVRvbSVTRUosVixewsbNFy7SiUcp5nj/Q7DeALcBnT/O8zucYOtX1XT3fGbWilETdC6yacr8LOHi6Y8wsBbQCQ+4+7u6DAO7+Y2A3cP70D3D3L7n7Fnff0tFRPV258+W5g8cZGpmomvHpqa7u7uCHLw1xckKXE62wUs5zzOxa4KPATe4+PtMb6XyOp3Q2RyphaFQyPKUk6m1At5mtM7N64DZg67RjtgJ3FG7fAnzX3d3MOgqTVDCz9UA3sKc8odeuYovzLVU0Pl109fntTGRy/PClwahDqTWznudmdhnwRfJJui+CGGUOMjnXjO9AzZqoC2POdwMPA88DD7r7c2Z2j5ndVDjs74A2M+sh38VdXNpxNfCsme0gP8ns9919qNy/RK15/MV+Np23iI6FDVGHUnZvXNdGfSrBE7s0Tl1JJZ7nnwVagK8VlltOr7BLjKWzOc34DlRJC3Dd/SHgoWmPfWzK7THg1hle9w3gG3OMUaY4Ppbmx/uOcOdb1kUdyrxYUJ/k8rVLeeyFPv70HRepm66CSjjPr614UFI2StThUqkFZuszB8nknBtfc17UocybGy5Zzp7+EXZoj2qRsslkXdf5DpQSdWAe3H6AC5cvZHNXa9ShzJt3XrqCxroED2w7MPvBIlKSCbWog6VSC8jzh47zbO8x3rNlVVV3CS9qrOPGS87jWzsOava3SJlksq6lWYFSog7Ig9sPUJc03nXZma5DUR3es2UVw+MZHvrpoahDEakKmZxa1KFSqQViPJPlX55+mes2LWdpc33U4cy7N65bytq2Jh7cru5vkXKYyDgpJeogqdQC8ejOPo6MpnnPG1bNfnAVMDNu3bKKH740xN6BkajDEQlevkWtru8QKVEH4oHtB1jR2shVG9ujDqVifu11XSQMtapFyiA/Rq2v/BCp1AJw8OhJntjVzy2v7yJZQ8srlrc2cs0FnXzjJ72TGwqIyLmZKFxCVMKjRB2Ar/+4F3e4dUttdHtP9Z4tqzh8fFwbdYjMUUbLs4KlUou5sXSWf/rRfq7Y0MaqpU1Rh1Nxb7uwk/aWev7+yb1RhyIStExOy7NCpUQdc3//5F4OHRvj7rdtjDqUSNSnEvze1Rt4YtcAT6hVLXLOJjI5zfoOlEotxoZGJvj8Yz287cJOrthQO5PIpvutK9bQtWQBn3ro52RzM26RLCKzUIs6XErUMfaX39nFyESGj9xwYdShRKohleRDb7+A5w8d55+ffjnqcESCpE05wqVSi6m9AyP83x/s471vWEX3soVRhxO5d25eweauVv78319gLK3LioqcrfymHPrKD5FKLaY++/AL1KcSfODa86MOJRYSCeO/33gRh46Ncd+TL0Udjkhw0tkc9Sl1fYdIiTqGfrL/CP/600P87lvW07moMepwYuNN69u49qJO/vqx3QwOj0cdjkhQ0tmcWtSBUqnFzImxNB/62g46FjZw19Xrow4ndj58w4WcTGf5b9/4KTlNLBMpWSbrpDSZLEhK1DGSyzkffHAHewdH+cvbLqO5IRV1SLGzsXMhf/qOi3j0+cP85Xd3RR2OSDDSuRz1mkwWJJVajPzVd3t4ZOdhPnrjRbx5Q1vU4cTWHVes5Vdft5K/eHQXj+w8HHU4IkFIq0UdLCXqmHh052E+9+iL/OrrVvLbV66NOpxYMzM+9e5LuGRlKx944Bl6+oajDkkk1nI5J5vTrO9QqdRiYOfB43zggWe4ZGUrn3r3JZip1jubxrokX/zN19OQSnDXP26n/4Qml4mcTjqX39SmPqWv/BCp1CL22M/7uPWu8HEqAAAJxElEQVQLT9HUkOQLv/l6GuuSUYcUjBWLF/D5//I6Dh49ybvufZIXD5+IOiSRWMpk8xMvtXtWmJSoI/TlJ1/izq9sY217M998/1WsXLwg6pCC88b1bTxw15uZyOb4tc8/xX++qOuBi0w3mag1mSxIKrUIjKWzfOybP+MT39rJ2y5cxoO/92aWt2q99Lm6dNVivvn+K1m5ZAG/8+Vt/P2TL2nplsgUE4X93Os1mSxIStQV5O58+9mD/PKf/yf/8P19/O5b1vHF33y9lmGVwYrFC/j6H1zBNed38D++tZN3ff5Jtu8dijoskVjIFMao1aIOkzJEhTy9/wifeuh5tu09wkXnLeJ/3XqplmCVWUtDir/5rS18c8fL/Nm/vcAtX/g+77x0BR+67gJWt9XeXt4iRcWub23KESYl6nl0bDTNvzzzMg9sO8DOQ8dpa67nU+++hPe+YRVJTeqYF4mE8e7Lunj7xcv5wn/s5ouP7+FbOw5y5cY23rNlFW+/eLkm7EnNKXZ9a5vLMJWUqM3seuB/A0ngb93909OebwD+AXg9MAi81933Fp77CHAnkAX+yN0fLlv0MZPLOS/2neCpnkGe2j3A47sGmMjkeM3KRdxz88W867KVLGqsizrMmtBUn+KD113Ar79xDQ9sO8DXfnyAP77/GVoX1HHNBR1csaGNKza0s2qpWtpFcznPJd5OzfpWizpEsyZqM0sC9wK/AvQC28xsq7vvnHLYncARd99oZrcBfwa818w2AbcBFwMrgEfN7Hx3D3qfQndnaGSCl4+e5MXDw7x4+AQv/OIEP3v5GIMjEwCsbWvi1y9fza1burh4RWvEEdeu5a2N/PG13fzh2zby/T2DfOPHvTy+a4BvPnMQgJWLF3DxikVcsHwh5y9byMbOFlYsXsCixlRNrWefy3le+WjlbKXVog5aKS3qy4Eed98DYGb3AzcDU0/gm4FPFG5/Hfg/lv+Wuxm4393HgZfMrKfwft8vT/hn5p6/Gk/WnVwuP6Eim3PSWSedzU3+jKVzjGeyjKVznJzIMjKRYXg8w8h4huMnMwyNTnBkZIKhkQkOHx/j0LExxjO5yc+pTyXo7mzhrRd28qb1bbx5Q5uWWsVMImFcubGdKze24+709A3z1O5BfvTSEC8cPsF3ft5HdspM8ZaGFOe1NtK5qIElTfUsba5nSVM9CxtTtDSkaG7I/9tQl6CxLklDKkFDKkl9MkFdyqhLJqhLJEgmjaQZyUT+J2HEtQJwzue5u2uKfcydStRqUYeolES9Ejgw5X4v8MbTHePuGTM7BrQVHv/BtNeuPOdogaGRCX7pM4/h+c8q/AuOk/P8YzmHnDvl+PpIJozFC+pY0lzPkqY6Ll7ZynUXL+e81kZWLF5Ad2cLa9qaNeYcEDOje9lCupct5I4r1gL5JXN7+kfY3T/MoWMnOXh0jINHTzIwPM7Bo8cZGpng2Ml0mT4fkmYkzMDIJ2+Mwl0SZixvbeSRD/5SWT6vRHM5zwfO9UP/57d38uD2A7MfKHNSrIQqUYeplEQ9UwaangJPd0wpr8XM7gLuAli9evUZg2lIJbhlS9crvtjM8l9uZla4nf8itCktmWTCSCXyX451qQT1SSOVSFCXStCYStBQl6QxlWBBfXKytdTckKKpLklCSbjqNdYl2bRiEZtWLDrtMdmcT/a0jIzne13GMznG0tnJXpl01skUemomsk4u52RyTjaXI5vLVyBzhZ4eJ3+fKRXLYsVzYWPF53nO5Tx/5UFncT5ftnoJGa15r4jmhiSvW7M46jDkHJTybdALrJpyvws4eJpjes0sBbQCQyW+Fnf/EvAlgC1btpzxrG1uSPHxd15cQtgi5ZVMGK0L6mhdUJUTAudynr/C2ZzP79h8Hu/YfN4cwhapfqX0g2wDus1snZnVk58ctnXaMVuBOwq3bwG+Wxi32grcZmYNZrYO6AZ+VJ7QRaSM5nKei8g8mrVFXRiLuht4mPyyjfvc/TkzuwfY7u5bgb8D/rEwWWyI/ElO4bgHyU9IyQDvD33Gt0g1mst5LiLzy+JWId6yZYtv37496jBEYs/MfuzuW6KO40x0PouU5kzns6YAioiIxJgStYiISIwpUYuIiMSYErWIiEiMxW4ymZn1A/tKOLSdOVwRKUKKu7KqOe417t5RiWDOVYnnczWXURyFGjeEG/uczufYJepSmdn2uM94nYnirizFHX+h/q6Ku/JCjX2ucavrW0REJMaUqEVERGIs5ET9pagDOEeKu7IUd/yF+rsq7soLNfY5xR3sGLWIiEgtCLlFLSIiUvWCStRmdquZPWdmOTPbMu25j5hZj5m9YGZvjyrGUpjZJ8zsZTN7pvBzY9QxnY6ZXV/4m/aY2YejjudsmNleM/tp4W8c2wtOm9l9ZtZnZj+b8thSM3vEzHYV/l0SZYzzoRrO55DOZQj3fK71czmoRA38DPhV4PGpD5rZJvI7+VwMXA983sySlQ/vrHzO3V9b+Hko6mBmUvgb3gvcAGwCbi/8rUPy1sLfOM5LOr5M/v/tVB8GvuPu3cB3CverTbWcz7E/l6EqzueaPZeDStTu/ry7vzDDUzcD97v7uLu/BPQAl1c2uqp0OdDj7nvcfQK4n/zfWsrI3R8nv23kVDcDXync/grwrooGVQE6nytO5/M8m69zOahEfQYrgQNT7vcWHouzu83s2UJXSVy7NUP8u07lwL+b2Y/N7K6ogzlLy9z9EEDh386I46mk0P7fhXAuQ3h/16lq+lxOlT2kOTKzR4HlMzz1UXf/5uleNsNjkU5nP9PvAfw18EnyMX4S+HPgdyoXXcli93c9S1e6+0Ez6wQeMbOfF2q8UiHVcD5XybkMMfu7nqWaPpdjl6jd/dpzeFkvsGrK/S7gYHkiOjel/h5m9jfAt+c5nHMVu7/r2XD3g4V/+8zsn8l3/YVych82s/Pc/ZCZnQf0RR3QuaiG87lKzmWI2d/1bNT6uVwtXd9bgdvMrMHM1gHdwI8ijum0CoVV9G7yk2riaBvQbWbrzKye/ASfrRHHVBIzazazhcXbwHXE9+88k63AHYXbdwCna31Wo2DO54DOZQj0fNa5HMMW9ZmY2buBvwI6gH81s2fc/e3u/pyZPQjsBDLA+909G2Wss/iMmb2WfLfTXuD3og1nZu6eMbO7gYeBJHCfuz8XcVilWgb8s5lB/v/5V939/0Ub0szM7J+Aa4B2M+sFPg58GnjQzO4E9gO3Rhfh/KiS8zmIcxmCPp9r/lzWlclERERirFq6vkVERKqSErWIiEiMKVGLiIjEmBK1iIhIjClRi4iIxJgStYiISIwpUYuIiMSYErWIiEiM/X/03AZpQ1zNcgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#implementation of sigmoid function\n",
    "def sigmoid(z):\n",
    "    value = 1/(1+np.exp(-z))\n",
    "    return value\n",
    "\n",
    "def relu(z):\n",
    "    value = np.maximum(0,z)\n",
    "    return value\n",
    "\n",
    "def derivative_of_activation(a,activation='sigmoid'):\n",
    "    derivative = 0\n",
    "    shape = a.shape\n",
    "    if(activation == 'sigmoid'):\n",
    "        derivative = sigmoid(a)*(1-sigmoid(a))\n",
    "    elif(activation == 'relu'):\n",
    "        derivative = a>0\n",
    "        derivative = derivative.astype(np.int32)\n",
    "    return derivative.reshape(shape)\n",
    "\n",
    "#plot functions' shape\n",
    "plt.figure(figsize=(8,8))\n",
    "\n",
    "plt.subplot(221)\n",
    "plt.title('Sigmoid function\\'s shape')\n",
    "z = np.linspace(-10,10)\n",
    "plt.plot(z,sigmoid(z))\n",
    "\n",
    "plt.subplot(222)\n",
    "plt.title('ReLu function\\'s shape')\n",
    "z = np.linspace(-10,10)\n",
    "plt.plot(z,relu(z))\n",
    "\n",
    "plt.subplot(223)\n",
    "plt.title('Sigmoid function\\'s derivative \\'s shape')\n",
    "z = np.linspace(-10,10)\n",
    "plt.plot(z,derivative_of_activation(z))\n",
    "\n",
    "plt.subplot(224)\n",
    "plt.title('ReLu\\'s derivative \\'s shape')\n",
    "z = np.linspace(-10,10)\n",
    "plt.plot(z,derivative_of_activation(z,activation='relu'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note the dimensions of vectors:\n",
    "# w - (number_of_units_in_current_layer,number_of_units_in_previous_layer)\n",
    "# b - (number_of_units_in_current_layer,1) *broadcasted*\n",
    "# X - (x_dims,m)                           *m=number of samples\n",
    "\n",
    "'''\n",
    "Note that in neural networks with multiple layers, when doing forward prop, we need to cache some values for back prop step.\n",
    "For example:\n",
    "    when doing forward prop, we calculate z = np.dot(w,X)+b ,and then output a = g(z), when function g can be relu or sigmoid.\n",
    "    when doing back prop, we calculate dz = da*g'(z), where g'(z) is the derivative of g(z)\n",
    "    Therefore we can just cache z to save some time for calculating z multiple times.\n",
    "    \n",
    "In a nutshell, what we can cache in forward prop are: z,a,w\n",
    "'''\n",
    "\n",
    "# Forward propagation step: compute the predicted y's label\n",
    "def linear_forward_prop(w,b,X):\n",
    "    z = np.dot(w,X) + b\n",
    "    linear_cache = w\n",
    "    #print('Linear_forward_z=',z)\n",
    "    return z,linear_cache\n",
    "\n",
    "def single_layer_forward_prop(z,activation='relu'):\n",
    "    if(activation == 'relu'):\n",
    "        a = relu(z)\n",
    "    elif(activation == 'sigmoid'):\n",
    "        a = sigmoid(z)\n",
    "    activation_cache = a,z\n",
    "    return a,activation_cache\n",
    "\n",
    "def L_layer_forward_prop(X,parameters):\n",
    "    A = {0:X}\n",
    "    cache = {}  \n",
    "    L = len(parameters) // 2\n",
    "    for i in range(L-1):\n",
    "        w = parameters['w'+str(i+1)]\n",
    "        b = parameters['b'+str(i+1)]\n",
    "        z,linear_cache = linear_forward_prop(w,b,A[i])\n",
    "        a,activation_cache = single_layer_forward_prop(z,activation='relu')\n",
    "        A[i+1] = a\n",
    "        cache['layer_'+str(i+1)] = linear_cache,activation_cache\n",
    "    w = parameters['w'+str(L)]\n",
    "    b = parameters['b'+str(L)]    \n",
    "    z,linear_cache = linear_forward_prop(w,b,A[L-1])\n",
    "    yhat,activation_cache = single_layer_forward_prop(z,activation='sigmoid')\n",
    "    cache['layer_'+str(L)] = linear_cache,activation_cache\n",
    "    return yhat,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute cost function: used to check convergence\n",
    "def compute_cost(yhat,y):\n",
    "    m = yhat.shape[1]\n",
    "    cost = -np.sum(y*np.log(yhat)+(1-y)*np.log(1-yhat))/m\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Back propagation step: compute partial derivatives of each parameter respectively\n",
    "def linear_back_prop(w,a_previous,dz_l):\n",
    "    m = a_previous.shape[1]\n",
    "    dw_l = np.dot(dz_l,a_previous.T)/m\n",
    "    assert(dw_l.shape == w.shape)\n",
    "    \n",
    "    db = np.sum(dz_l,axis=1,keepdims=True)/m\n",
    "    da_previous = np.dot(w.T,dz_l)\n",
    "    assert(da_previous.shape == a_previous.shape)\n",
    "    return dw_l,db,da_previous\n",
    " # Note: dw should have the same dimension as w have.Therefore back_prop returns dw.T\n",
    "    \n",
    "def single_layer_back_prop(da_l,z_l,activation):\n",
    "    derivative = derivative_of_activation(z_l,activation)\n",
    "    dz_l = da_l * derivative\n",
    "    assert(dz_l.shape == z_l.shape)\n",
    "    return dz_l\n",
    "\n",
    "def L_layer_back_prop(X,y,cache_from_forward,testAL = np.zeros(1)):\n",
    "    dW = {}\n",
    "    db = {}\n",
    "    dA = {}\n",
    "    W = {}\n",
    "    A = {0:X}\n",
    "    Z = {}\n",
    "    m = y.shape[1]\n",
    "    L=len(cache_from_forward)\n",
    "    for layer in range(1,L+1):\n",
    "        linear_cache_l,activation_cache_l = cache_from_forward['layer_'+str(layer)]\n",
    "        W[layer] = linear_cache_l\n",
    "        A[layer],Z[layer] = activation_cache_l\n",
    "    #Initialize the output layer\n",
    "    if((testAL == np.zeros(1))):\n",
    "        yhat = A[L]\n",
    "    else:\n",
    "        yhat = testAL\n",
    "    dA[L] = -np.divide(y,yhat)+np.divide((1-y),(1-yhat))\n",
    "    dz_L = single_layer_back_prop(dA[L],Z[L],activation='sigmoid')\n",
    "    dW[L],db[L],dA[L-1] = linear_back_prop(W[L],A[L-1],dz_L)\n",
    "    \n",
    "    for i in reversed(range(1,L)):\n",
    "        dz_l = single_layer_back_prop(dA[i],Z[i],activation='relu')\n",
    "        dW[i],db[i],dA[i-1] = linear_back_prop(W[i],A[i-1],dz_l)\n",
    "        assert(dW[i].shape == W[i].shape)\n",
    "    return dW,db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(params,learning_rate,dW,dB):\n",
    "        L = len(params) // 2\n",
    "        for j in range(1,L+1):\n",
    "            params['w'+str(j)] = params['w'+str(j)] - learning_rate*dW[j]\n",
    "            params['b'+str(j)] = params['b'+str(j)] - learning_rate*dB[j]\n",
    "        return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dimension_convert(X,y):\n",
    "    # Dimension convert: make sure all vectors are in proper shapes.\n",
    "    y = y.reshape(1,-1)# y is a row vector\n",
    "    M = y.shape[1]  #  m = total number of trainning examples\n",
    "    if(X.shape[1] != M):\n",
    "        X=X.T       #=====> Note that array.reshape and array.T are different!\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The overall implementation of training a logistic regression\n",
    "# Note: net_structure indicates the shape of hidden layers and output layers. No input layer should be included.\n",
    "\n",
    "def train_neural_network(X,y,net_structure,number_of_iteration = 1000,learning_rate = 0.03,lambd = 0.7,print_cost = True,plot_cost = True):\n",
    "    # Dimension convert: make sure all vectors are in proper shapes.\n",
    "    X,y = dimension_convert(X,y)\n",
    "    print('*******Dimension Check*******')\n",
    "    print('Input feature\\'s dimension: ',X.shape)\n",
    "    print('Output\\'s dimension: ',y.shape)\n",
    "    print('*****************************')\n",
    "    \n",
    "    #Normalize input feature X\n",
    "    miu = np.sum(X)/(X.shape[0]*X.shape[1])\n",
    "    X = X - miu\n",
    "    sigma2 = np.sum(X**2)/(X.shape[0]*X.shape[1])\n",
    "    sigma = np.sqrt(sigma2)\n",
    "    X = X / sigma\n",
    "    \n",
    "    x_dim = X.shape[0]\n",
    "    L = len(net_structure) # number of layers\n",
    "    # Initialize parameters\n",
    "    nn_structure = [x_dim]+net_structure\n",
    "    params = nn_parameter_initialize(nn_structure)\n",
    "    print('Training {} layers neural network...'.format(len(params)//2))\n",
    "    if(plot_cost == True):\n",
    "        i_curve = []\n",
    "        cost_curve = []\n",
    "        plt.figure(figsize=(5,5))\n",
    "        plt.title('Cross entropy')\n",
    "    \n",
    "    cache={}\n",
    "    for i in range(1,number_of_iteration+1):\n",
    "            # Steps:\n",
    "                # 1:Forward propagation\n",
    "                # 2:Compute cost\n",
    "                # 3:Backward Propagation\n",
    "                # 4:Update parameters\n",
    "                \n",
    "        yhat,cache = L_layer_forward_prop(X,params)\n",
    "        assert(yhat.shape == y.shape)\n",
    "        cost = compute_cost(yhat,y)\n",
    "        dW,dB = L_layer_back_prop(X,y,cache)\n",
    "            \n",
    "        #Gradient decent\n",
    "        params = update_parameters(params,learning_rate,dW,dB)\n",
    "            \n",
    "        # Visualize the process of regression\n",
    "        if(i%100 == 0 and print_cost == True):\n",
    "            print('number of iterations:{}, cost = {}'.format(i,cost))\n",
    "        if(i%100 == 0 and plot_cost == True):\n",
    "            i_curve.append(i)\n",
    "            cost_curve.append(cost)\n",
    "            \n",
    "    if(plot_cost==True):        \n",
    "        i_curve = np.reshape(i_curve,(1,-1))\n",
    "        cost_curve = np.reshape(cost_curve,(1,-1))\n",
    "        plt.scatter(i_curve,cost_curve)\n",
    "    return params,miu,sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [],
   "source": [
    "#After training the neural network, we can now use it to make predictions.\n",
    "def nn_predict(parameters,miu,sigma,X,y=0,evaluate = True):\n",
    "    L = len(parameters) // 2\n",
    "    w1 = parameters['w1'] \n",
    "    num_of_features = w1.shape[1]\n",
    "    if(X.shape[0] != num_of_features):\n",
    "        X=X.T       #=====> Note that array.reshape and array.T are different!\n",
    "    X = X - miu\n",
    "    X = X / sigma\n",
    "    yhat,cache = L_layer_forward_prop(X,parameters)\n",
    "    yhat = yhat>0.5\n",
    "    #Codes below is used to evaluate the performance \n",
    "    #You can just ignore this part\n",
    "    if(evaluate == True):\n",
    "        y=y.reshape(1,-1)\n",
    "        train_accuracy = np.sum(yhat==y)/y.shape[1]\n",
    "        print('accuracy = %.2f\\n'%train_accuracy)\n",
    "    return yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Goal:Wanna classify whether our patient's breast cancer is malignant(0) or benign(1)\n"
     ]
    }
   ],
   "source": [
    "print(\"Goal:Wanna classify whether our patient's breast cancer is {}(0) or {}(1)\".format(dataset.target_names[0],dataset.target_names[1]))\n",
    "y = dataset['target']\n",
    "X = dataset['data']\n",
    "\n",
    "#Split up dataset in order to train as well as test the model\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3,random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum of dataset:  12.160083487500223\n",
      "Minimum of dataset:  -3.155175505366627\n",
      "Mean of dataset:  1.2496982789750506e-16\n",
      "*******Dimension Check*******\n",
      "Input feature's dimension:  (30, 398)\n",
      "Output's dimension:  (1, 398)\n",
      "*****************************\n",
      "Training 3 layers neural network...\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAE/CAYAAAAzEcqDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAXY0lEQVR4nO3df5CdVX3H8ffHJMRVW8KP1SFLYkJJU6PURrcRK1qGVBKclmQoToNaU0ubdsZMtdhoYlU0OgWMFXUaW9NCYaAaLKVpiujWktraFmM2piVE3GFFJJtFWQwB0dX84Ns/nrPpzc3d7LObu7k/zuc188ze5zzn3uececKHc55nz11FBGZmOXtOoxtgZtZoDkIzy56D0Myy5yA0s+w5CM0sew5CM8ueg9DMsucgtBOS9CZJvZKekfSYpC9KuqjR7ZoISbdI+kij22HNx0Foo5J0DfAJ4M+AFwGzgU8Dy0apP/XUta7+Wr39dhIiwpu34zbgdOAZ4I0nqPNB4E7gduBp4PeA6RThOZi2TwDTU/2zgbuBA8B+4KvAc9Kx9wD7gB8CfcDiUc45HfgY8CjwfeCvgI507GJgAHgX8DjwGPC2dGwVcAg4mPr1z6n8kXTu+4GfAlOBlwBfSe3cA1xecf5b0jm/nNr678CL07GNwJ9XtfefgXc2+np6G+Pfe6Mb4K05N2ApcBiYeoI6H0zhspxidtEBrAe+BrwQ6AT+G/hwqn9dCpFpaXstIGA+sBeYmerNAX5ulHN+AtgKnAn8TAqa69Kxi1Ob16fPfwPwY+CMdPwW4CNVn/cI8D/ArNT+aUA/8F7gNOCSFHjzKz7jh8DrUih/EvjPdGwRRfiPhPvZ6fwvavT19HbizVNjG81ZwBMRcXiMevdFxJaIeDYihoE3A+sj4vGIGAI+BPx2qnsIOIdiBHUoIr4aRWIcoQiVBZKmRcQjEfHt6hNJEvD7wB9HxP6I+CHFtH1FRbVD6fyHIuIeitHf/DH68KmI2JvafyHwAuD6iDgYEdsoRrFXVdT/QkT8R0T8FPhT4NWSZkXE14GngMWp3grgKxHx/THObw3mILTR/AA4u8R9s71V+zOB71bsfzeVAWygGG39i6SHJa0FiIh+4J0UI8zHJW2WNJPjdQLPA3ZKOiDpAPClVH603VXh/WOKYCvbh5nA3oh4tqoPXbXqR8QzFNP8kfbeCrwlvX4LcNsY57Ym4CC00dwH/IRi2nsi1V9fNAi8uGJ/diojIn4YEe+KiPOA3wCukbQ4HftsRFyU3hvADTXO9QQwDLw0Imak7fSIGCvoRmtrrfJBYJakyv82ZlPcvxwxa+SFpBdQTNMHU9HtwDJJL6e417ilZNusgRyEVlNEPAV8ANgoabmk50maJukySR89wVs/B7xPUqeks9Nn3A4g6dclnZ+muE9TTImPSJov6RJJ0ynCdzgdq27Ts8BfAzdKemH6zC5JS0p26/vAeWPU2Q78CHh36u/FFKG9uaLOGyRdJOk04MPA9ojYm9o4AOygGAn+Q5puW5NzENqoIuLjwDXA+4Ahiinhak48yvkI0EvxFHY38I1UBjAP+FeK+3b3AZ+OiK9Q3B+8nmLE9z2KBy3vHeXz30Mxvf6apKfT5411D3DETRT3IQ9IqtmHiDgIXA5cltrzaeCtEfGtimqfBa6lmBK/kuK+aKVbgQvwtLhlqLhXbWZlSLoFGIiI952gzusoRsFzqu41WpPyiNCsjiRNA94B/I1DsHU4CM3qRNJLKH4J+xyK33e0FuGpsZllzyNCM8ueg9DMstd037Zx9tlnx5w5cxrdDDNrMzt37nwiIjprHWu6IJwzZw69vb2NboaZtRlJ3x3tmKfGZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlr2pjW7Aydiyax8bevoYPDDMzBkdrFkyn+ULuxrdLDNrMS0bhFt27WPdXbsZPnQEgH0Hhll3124Ah6GZjUvLTo039PQdDcERw4eOsKGnr0EtMrNWVSoIJS2V1CepX9LaGsdfJ+kbkg5LurLq2EpJD6VtZb0aPnhgeFzlZmajGTMIJU0BNgKXAQuAqyQtqKr2KPA7wGer3nsmcC3wKmARcK2kM06+2TBzRse4ys3MRlNmRLgI6I+IhyPiILAZWFZZISIeiYj7gWer3rsE+HJE7I+IJ4EvA0vr0G7WLJlPx7Qpx5R1TJvCmiXz6/HxZpaRMkHYBeyt2B9IZWWUeq+kVZJ6JfUODQ2V+uDlC7u47ooL6JrRgYCuGR1cd8UFflBiZuNW5qmxapRFyc8v9d6I2ARsAuju7i772Sxf2OXgM7OTVmZEOADMqtg/Fxgs+fkn814zs1OiTBDuAOZJmivpNGAFsLXk5/cAl0o6Iz0kuTSVmZk1jTGDMCIOA6spAuxB4PMRsUfSekmXA0j6ZUkDwBuBz0jak967H/gwRZjuANanMjOzpqGI0rfkTonu7u7o7e1tdDPMrM1I2hkR3bWOtezKEjOzenEQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWWvVBBKWiqpT1K/pLU1jk+XdEc6vl3SnFQ+TdKtknZLelDSuvo238zs5I0ZhJKmABuBy4AFwFWSFlRVuxp4MiLOB24EbkjlbwSmR8QFwCuBPxgJSTOzZlFmRLgI6I+IhyPiILAZWFZVZxlwa3p9J7BYkoAAni9pKtABHASerkvLzczqpEwQdgF7K/YHUlnNOhFxGHgKOIsiFH8EPAY8CnwsIvafZJvNzOqqTBCqRlmUrLMIOALMBOYC75J03nEnkFZJ6pXUOzQ0VKJJZmb1UyYIB4BZFfvnAoOj1UnT4NOB/cCbgC9FxKGIeBz4L6C7+gQRsSkiuiOiu7Ozc/y9MDM7CWWCcAcwT9JcSacBK4CtVXW2AivT6yuBbRERFNPhS1R4PnAh8K36NN3MrD7GDMJ0z2810AM8CHw+IvZIWi/p8lTtJuAsSf3ANcDIr9hsBF4APEARqH8bEffXuQ9mZidFxcCteXR3d0dvb2+jm2FmbUbSzog47tYceGWJmZmD0MzMQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpa9qY1uQCvZsmsfG3r6GDwwzMwZHaxZMp/lC7sa3SwzO0kOwpK27NrHurt2M3zoCAD7Dgyz7q7dAA5DsxbnqXFJG3r6jobgiOFDR9jQ09egFplZvTgISxo8MDyucjNrHaWCUNJSSX2S+iWtrXF8uqQ70vHtkuZUHPtFSfdJ2iNpt6Tn1q/5p87MGR3jKjez1jFmEEqaAmwELgMWAFdJWlBV7WrgyYg4H7gRuCG9dypwO/CHEfFS4GLgUN1afwqtWTKfjmlTjinrmDaFNUvmN6hFZlYvZUaEi4D+iHg4Ig4Cm4FlVXWWAbem13cCiyUJuBS4PyL+FyAifhARR2hByxd2cd0VF9A1owMBXTM6uO6KC/ygxKwNlHlq3AXsrdgfAF41Wp2IOCzpKeAs4OeBkNQDdAKbI+KjJ93qBlm+sMvBZ9aGygShapRFyTpTgYuAXwZ+DNwraWdE3HvMm6VVwCqA2bNnl2iSmVn9lJkaDwCzKvbPBQZHq5PuC54O7E/l/x4RT0TEj4F7gFdUnyAiNkVEd0R0d3Z2jr8XZmYnoUwQ7gDmSZor6TRgBbC1qs5WYGV6fSWwLSIC6AF+UdLzUkD+KvDN+jTdzKw+xpwap3t+qylCbQpwc0TskbQe6I2IrcBNwG2S+ilGgivSe5+U9HGKMA3gnoj4wiT1xcxsQlQM3JpHd3d39Pb2NroZZtZm0vOJ7lrHvLLEzLLnIDSz7DkIzSx7DkIzy56D0Myy5yA0s+w5CM0sew5CM8ueg9DMsucgNLPsOQjNLHsOQjPLnoPQzLLnP/DexLbs2seGnj4GDwwzc0YHa5bM958KMJsEDsImtWXXPtbdtfvoH5Xfd2CYdXftBnAYmtWZp8ZNakNP39EQHDF86Agbevoa1CKz9uUgbFKDB4bHVW5mE+cgbFIzZ3SMq9zMJs5B2KTWLJlPx7Qpx5R1TJvCmiXzG9Qis/blhyVNauSBiJ8am00+B2ETW76wy8Fndgp4amxm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9/x6hHcNf/WU5chDaUf7qL8uVp8Z2lL/6y3LlILSj/NVflisHoR3lr/6yXDkI7Sh/9Zflyg9L7Ch/9ZflykFox/BXf1mOPDU2s+yVCkJJSyX1SeqXtLbG8emS7kjHt0uaU3V8tqRnJP1JfZptZlY/YwahpCnARuAyYAFwlaQFVdWuBp6MiPOBG4Ebqo7fCHzx5JtrZlZ/ZUaEi4D+iHg4Ig4Cm4FlVXWWAbem13cCiyUJQNJy4GFgT32abGZWX2UelnQBeyv2B4BXjVYnIg5Lego4S9Iw8B7g9YCnxXYcr222ZlAmCFWjLErW+RBwY0Q8kwaItU8grQJWAcyePbtEk6wdeG2zNYsyU+MBYFbF/rnA4Gh1JE0FTgf2U4wcPyrpEeCdwHslra4+QURsiojuiOju7OwcdyesNXltszWLMiPCHcA8SXOBfcAK4E1VdbYCK4H7gCuBbRERwGtHKkj6IPBMRPxFHdptbcBrm61ZjDkijIjDwGqgB3gQ+HxE7JG0XtLlqdpNFPcE+4FrgON+xcasmtc2W7MotbIkIu4B7qkq+0DF658AbxzjMz44gfZZG1uzZP4x9wjBa5utMbzEzhrGa5utWTgIraG8ttmagdcam1n2HIRmlj0HoZllz0FoZtnzwxLLitc2Wy0OQsuG1zbbaDw1tmx4bbONxkFo2fDaZhuNg9Cy4bXNNhoHoWXDf7fZRuOHJZYNr2220TgILSte22y1eGpsZtlzEJpZ9jw1NptEXsnSGhyEZpPEK1lah6fGZpPEK1lah4PQbJJ4JUvrcBCaTRKvZGkdDkKzSeKVLK3DD0vMJolXsrQOB6HZJPJKltbgqbGZZc9BaGbZcxCaWfZ8j9CsjXhJ38Q4CM3ahJf0TZynxmZtwkv6Js5BaNYmvKRv4hyEZm3CS/omzkFo1ia8pG/i/LDErE14Sd/EOQjN2oiX9E2Mp8Zmlj0HoZllr1QQSloqqU9Sv6S1NY5Pl3RHOr5d0pxU/npJOyXtTj8vqW/zzayRtuzax2uu38bctV/gNddvY8uufY1u0oSMeY9Q0hRgI/B6YADYIWlrRHyzotrVwJMRcb6kFcANwG8BTwC/ERGDkl4G9AC+gWHWBtppJUuZEeEioD8iHo6Ig8BmYFlVnWXAren1ncBiSYqIXRExmMr3AM+VNL0eDTezxmqnlSxlgrAL2FuxP8Dxo7qjdSLiMPAUcFZVnd8EdkXETyfWVDNrJu20kqVMEKpGWYynjqSXUkyX/6DmCaRVknol9Q4NDZVokpk1WjutZCkThAPArIr9c4HB0epImgqcDuxP++cC/wi8NSK+XesEEbEpIrojoruzs3N8PTCzhminlSxlgnAHME/SXEmnASuArVV1tgIr0+srgW0REZJmAF8A1kXEf9Wr0WbWeMsXdnHdFRfQNaMDAV0zOrjuigta7kEJlHhqHBGHJa2meOI7Bbg5IvZIWg/0RsRW4CbgNkn9FCPBFentq4HzgfdLen8quzQiHq93R8zs1GuXlSyKqL7d11jd3d3R29vb6GaYWZuRtDMiumsd88oSM8uev3TBzFrKZPxdFgehmbWMyVrN4qmxmbWMyVrN4iA0s5YxWatZHIRm1jImazWLg9DMWsZkrWbxwxIzaxmT9XdZHIRm1lImYzWLp8Zmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2XMQmln2HIRmlj0HoZllz0FoZtlzEJpZ9koFoaSlkvok9UtaW+P4dEl3pOPbJc2pOLYulfdJWlK/ppuZ1ceYQShpCrARuAxYAFwlaUFVtauBJyPifOBG4Ib03gXACuClwFLg0+nzzMyaRpkR4SKgPyIejoiDwGZgWVWdZcCt6fWdwGJJSuWbI+KnEfEdoD99nplZ0ygThF3A3or9gVRWs05EHAaeAs4q+V4zs4YqE4SqURYl65R5L5JWSeqV1Ds0NFSiSWZm9TO1RJ0BYFbF/rnA4Ch1BiRNBU4H9pd8LxGxCdgEIGlI0nfLdqBBzgaeaHQjJlm799H9a33j7eOLRztQJgh3APMkzQX2UTz8eFNVna3ASuA+4EpgW0SEpK3AZyV9HJgJzAO+fqKTRURniTY1lKTeiOhudDsmU7v30f1rffXs45hBGBGHJa0GeoApwM0RsUfSeqA3IrYCNwG3SeqnGAmuSO/dI+nzwDeBw8DbI+JIPRpuZlYvijjulp2Nwf+3bX3uX+urZx+9smRiNjW6AadAu/fR/Wt9deujR4Rmlj2PCM0sew7CGiTNkvRvkh6UtEfSO1L5mZK+LOmh9POMVC5Jn0prqu+X9IrG9qAcSVMk7ZJ0d9qfm9aKP5TWjp+WykddS96sJM2QdKekb6Xr+Oo2vH5/nP59PiDpc5Ke28rXUNLNkh6X9EBF2bivmaSVqf5DklaWObeDsLbDwLsi4iXAhcDb07rptcC9ETEPuDftQ7EOe17aVgF/eeqbPCHvAB6s2L8BuDH170mKNeQwylryJvdJ4EsR8QvAyyn62TbXT1IX8EdAd0S8jOI3OlbQ2tfwForvJKg0rmsm6UzgWuBVFMt5rx0JzxOKCG9jbMA/Aa8H+oBzUtk5QF96/Rngqor6R+s160bxy+33ApcAd1OsAnoCmJqOvxroSa97gFen11NTPTW6Dyfo288C36luY5tdv5Hlq2ema3I3sKTVryEwB3hgotcMuAr4TEX5MfVG2zwiHEOaQiwEtgMviojHANLPF6Zqrbim+hPAu4Fn0/5ZwIEo1orDsX0YbS15szoPGAL+Nk39/0bS82mj6xcR+4CPAY8Cj1Fck520zzUcMd5rNqFr6SA8AUkvAP4BeGdEPH2iqjXKmvZxvKRfBx6PiJ2VxTWqRoljzWgq8ArgLyNiIfAj/n9KVUur9Y803VsGzKVYtfV8iulitVa9hmM5qe83qOYgHIWkaRQh+HcRcVcq/r6kc9Lxc4DHU3mpNdVN5DXA5ZIeofhatUsoRogz0lpxOLYPR/tXtZa8WQ0AAxGxPe3fSRGM7XL9AH4N+E5EDEXEIeAu4Fdon2s4YrzXbELX0kFYgyRRLBt8MCI+XnFoZE016ec/VZS/NT3JuhB4amQ434wiYl1EnBsRcyhusG+LiDcD/0axVhyO799Iv4+uJT+FTR6XiPgesFfS/FS0mGKZZ1tcv+RR4EJJz0v/Xkf62BbXsMJ4r1kPcKmkM9Ko+dJUdmKNvjnajBtwEcVw+n7gf9L2Bop7KvcCD6WfZ6b6ovgW728Duyme5DW8HyX7ejFwd3p9HsWXYvQDfw9MT+XPTfv96fh5jW53iX79EtCbruEW4Ix2u37Ah4BvAQ8AtwHTW/kaAp+juN95iGJkd/VErhnwu6mf/cDbypzbK0vMLHueGptZ9hyEZpY9B6GZZc9BaGbZcxCaWfYchGaWPQehmWXPQWhm2fs/5JDYa68Oo2cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Train the neural network\n",
    "print('Maximum of dataset: ',np.max(X_train))\n",
    "print('Minimum of dataset: ',np.min(X_train))\n",
    "print('Mean of dataset: ',np.mean(X_train))\n",
    "parameters,miu,sigma = train_neural_network(X_train,y_train,[3,10,1],number_of_iteration = 1000,learning_rate = 0.1,print_cost = False,plot_cost = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy:\n",
      "accuracy = 0.99\n",
      "\n",
      "Accuracy in test sets:\n",
      "accuracy = 0.95\n",
      "\n",
      "sklearn's neural network training accuracy:\n",
      "1.00\n",
      "sklearn's neural network training accuracy:\n",
      "0.95\n",
      "Loss =  0.009620371001531857\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the performance of the unit on training set and test set\n",
    "print('Training accuracy:')\n",
    "Yhat = nn_predict(parameters,miu,sigma,X_train,y_train,evaluate = True)\n",
    "print('Accuracy in test sets:')\n",
    "Ypredict = nn_predict(parameters,miu,sigma,X_test,y_test,evaluate = True)\n",
    "\n",
    "# Use sklearn's standard scaler function to uniform inout data\n",
    "ss = StandardScaler()\n",
    "X_train = ss.fit_transform(X_train)\n",
    "X_test = ss.fit_transform(X_test)\n",
    "\n",
    "mlp = MLPClassifier(solver='lbfgs', alpha=0.03,hidden_layer_sizes=(3,8), random_state=1)\n",
    "mlp = mlp.fit(X_train,y_train)\n",
    "train_score=mlp.score(X_train,y_train)#How many samples can the model predict right? \n",
    "print('sklearn\\'s neural network training accuracy:')\n",
    "print('%.2f'%train_score)\n",
    "#Okay, we have built our own neural network. Let's compare our unit with sklearn's! \n",
    "test_score=mlp.score(X_test,y_test)\n",
    "print('sklearn\\'s neural network training accuracy:')\n",
    "print('%.2f'%test_score)\n",
    "print('Loss = ',mlp.loss_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
